32#include "llvm/ADT/APFloat.h"
33#include "llvm/ADT/APInt.h"
34#include "llvm/ADT/FloatingPointMode.h"
35#include "llvm/ADT/SmallPtrSet.h"
36#include "llvm/ADT/StringExtras.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/InlineAsm.h"
40#include "llvm/IR/Intrinsics.h"
41#include "llvm/IR/IntrinsicsAArch64.h"
42#include "llvm/IR/IntrinsicsAMDGPU.h"
43#include "llvm/IR/IntrinsicsARM.h"
44#include "llvm/IR/IntrinsicsBPF.h"
45#include "llvm/IR/IntrinsicsHexagon.h"
46#include "llvm/IR/IntrinsicsLoongArch.h"
47#include "llvm/IR/IntrinsicsNVPTX.h"
48#include "llvm/IR/IntrinsicsPowerPC.h"
49#include "llvm/IR/IntrinsicsR600.h"
50#include "llvm/IR/IntrinsicsRISCV.h"
51#include "llvm/IR/IntrinsicsS390.h"
52#include "llvm/IR/IntrinsicsVE.h"
53#include "llvm/IR/IntrinsicsWebAssembly.h"
54#include "llvm/IR/IntrinsicsX86.h"
55#include "llvm/IR/MDBuilder.h"
56#include "llvm/IR/MatrixBuilder.h"
57#include "llvm/Support/ConvertUTF.h"
58#include "llvm/Support/ScopedPrinter.h"
59#include "llvm/TargetParser/AArch64TargetParser.h"
60#include "llvm/TargetParser/X86TargetParser.h"
65using namespace CodeGen;
69 Align AlignmentInBytes) {
71 switch (CGF.
getLangOpts().getTrivialAutoVarInit()) {
72 case LangOptions::TrivialAutoVarInitKind::Uninitialized:
75 case LangOptions::TrivialAutoVarInitKind::Zero:
76 Byte = CGF.
Builder.getInt8(0x00);
78 case LangOptions::TrivialAutoVarInitKind::Pattern: {
80 Byte = llvm::dyn_cast<llvm::ConstantInt>(
88 I->addAnnotationMetadata(
"auto-init");
103 static SmallDenseMap<unsigned, StringRef, 64> F128Builtins{
104 {Builtin::BI__builtin___fprintf_chk,
"__fprintf_chkieee128"},
105 {Builtin::BI__builtin___printf_chk,
"__printf_chkieee128"},
106 {Builtin::BI__builtin___snprintf_chk,
"__snprintf_chkieee128"},
107 {Builtin::BI__builtin___sprintf_chk,
"__sprintf_chkieee128"},
108 {Builtin::BI__builtin___vfprintf_chk,
"__vfprintf_chkieee128"},
109 {Builtin::BI__builtin___vprintf_chk,
"__vprintf_chkieee128"},
110 {Builtin::BI__builtin___vsnprintf_chk,
"__vsnprintf_chkieee128"},
111 {Builtin::BI__builtin___vsprintf_chk,
"__vsprintf_chkieee128"},
112 {Builtin::BI__builtin_fprintf,
"__fprintfieee128"},
113 {Builtin::BI__builtin_printf,
"__printfieee128"},
114 {Builtin::BI__builtin_snprintf,
"__snprintfieee128"},
115 {Builtin::BI__builtin_sprintf,
"__sprintfieee128"},
116 {Builtin::BI__builtin_vfprintf,
"__vfprintfieee128"},
117 {Builtin::BI__builtin_vprintf,
"__vprintfieee128"},
118 {Builtin::BI__builtin_vsnprintf,
"__vsnprintfieee128"},
119 {Builtin::BI__builtin_vsprintf,
"__vsprintfieee128"},
120 {Builtin::BI__builtin_fscanf,
"__fscanfieee128"},
121 {Builtin::BI__builtin_scanf,
"__scanfieee128"},
122 {Builtin::BI__builtin_sscanf,
"__sscanfieee128"},
123 {Builtin::BI__builtin_vfscanf,
"__vfscanfieee128"},
124 {Builtin::BI__builtin_vscanf,
"__vscanfieee128"},
125 {Builtin::BI__builtin_vsscanf,
"__vsscanfieee128"},
126 {Builtin::BI__builtin_nexttowardf128,
"__nexttowardieee128"},
132 static SmallDenseMap<unsigned, StringRef, 4> AIXLongDouble64Builtins{
133 {Builtin::BI__builtin_frexpl,
"frexp"},
134 {Builtin::BI__builtin_ldexpl,
"ldexp"},
135 {Builtin::BI__builtin_modfl,
"modf"},
141 if (FD->
hasAttr<AsmLabelAttr>())
147 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad() &&
148 F128Builtins.find(BuiltinID) != F128Builtins.end())
149 Name = F128Builtins[BuiltinID];
152 &llvm::APFloat::IEEEdouble() &&
153 AIXLongDouble64Builtins.find(BuiltinID) !=
154 AIXLongDouble64Builtins.end())
155 Name = AIXLongDouble64Builtins[BuiltinID];
160 llvm::FunctionType *Ty =
163 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
169 QualType T, llvm::IntegerType *IntType) {
172 if (
V->getType()->isPointerTy())
173 return CGF.
Builder.CreatePtrToInt(
V, IntType);
175 assert(
V->getType() == IntType);
180 QualType T, llvm::Type *ResultType) {
183 if (ResultType->isPointerTy())
184 return CGF.
Builder.CreateIntToPtr(
V, ResultType);
186 assert(
V->getType() == ResultType);
198 if (Align % Bytes != 0) {
209 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
218 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
220 llvm::IntegerType *IntType =
223 llvm::Type *IntPtrType =
226 llvm::Value *Args[2];
227 Args[0] = CGF.
Builder.CreateBitCast(DestPtr, IntPtrType);
229 llvm::Type *ValueType = Args[1]->getType();
230 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
233 Kind, Args[0], Args[1], Ordering);
243 unsigned SrcAddrSpace =
Address->
getType()->getPointerAddressSpace();
245 Address, llvm::PointerType::get(Val->
getType(), SrcAddrSpace),
"cast");
247 LV.setNontemporal(
true);
256 LV.setNontemporal(
true);
261 llvm::AtomicRMWInst::BinOp Kind,
270 llvm::AtomicRMWInst::BinOp Kind,
272 Instruction::BinaryOps Op,
273 bool Invert =
false) {
282 llvm::IntegerType *IntType = llvm::IntegerType::get(
285 llvm::Value *Args[2];
287 llvm::Type *ValueType = Args[1]->getType();
288 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
292 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
297 llvm::ConstantInt::getAllOnesValue(IntType));
321 llvm::IntegerType *IntType = llvm::IntegerType::get(
327 llvm::Type *ValueType = Args[1]->
getType();
328 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
332 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
333 llvm::AtomicOrdering::SequentiallyConsistent);
336 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
359 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
373 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
374 AtomicOrdering::Monotonic :
382 Destination, Comparand, Exchange,
383 SuccessOrdering, FailureOrdering);
384 Result->setVolatile(
true);
398 AtomicOrdering SuccessOrdering) {
405 assert(Destination->getType()->isPointerTy());
406 assert(!ExchangeHigh->getType()->isPointerTy());
407 assert(!ExchangeLow->getType()->isPointerTy());
408 assert(ComparandPtr->getType()->isPointerTy());
411 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
412 ? AtomicOrdering::Monotonic
416 llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.
getLLVMContext(), 128);
417 Address ComparandResult(ComparandPtr, Int128Ty,
421 ExchangeHigh = CGF.
Builder.CreateZExt(ExchangeHigh, Int128Ty);
422 ExchangeLow = CGF.
Builder.CreateZExt(ExchangeLow, Int128Ty);
424 CGF.
Builder.CreateShl(ExchangeHigh, llvm::ConstantInt::get(Int128Ty, 64));
425 llvm::Value *Exchange = CGF.
Builder.CreateOr(ExchangeHigh, ExchangeLow);
431 SuccessOrdering, FailureOrdering);
437 CXI->setVolatile(
true);
444 Value *Success = CGF.
Builder.CreateExtractValue(CXI, 1);
449 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
456 ConstantInt::get(IntTy, 1),
458 return CGF.
Builder.CreateAdd(
Result, ConstantInt::get(IntTy, 1));
462 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
469 ConstantInt::get(IntTy, 1),
471 return CGF.
Builder.CreateSub(
Result, ConstantInt::get(IntTy, 1));
482 Load->setVolatile(
true);
492 llvm::StoreInst *Store =
494 Store->setVolatile(
true);
502 const CallExpr *E,
unsigned IntrinsicID,
503 unsigned ConstrainedIntrinsicID) {
506 if (CGF.
Builder.getIsFPConstrained()) {
507 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
508 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
509 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0 });
512 return CGF.
Builder.CreateCall(F, Src0);
519 const CallExpr *E,
unsigned IntrinsicID,
520 unsigned ConstrainedIntrinsicID) {
524 if (CGF.
Builder.getIsFPConstrained()) {
525 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
526 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
527 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
530 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
537 llvm::Intrinsic::ID ConstrainedIntrinsicID) {
541 if (CGF.
Builder.getIsFPConstrained()) {
542 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
544 {Src0->getType(), Src1->getType()});
545 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0, Src1});
550 return CGF.
Builder.CreateCall(F, {Src0, Src1});
556 const CallExpr *E,
unsigned IntrinsicID,
557 unsigned ConstrainedIntrinsicID) {
562 if (CGF.
Builder.getIsFPConstrained()) {
563 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
564 Function *F = CGF.
CGM.
getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
565 return CGF.
Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
568 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
575 unsigned IntrinsicID,
576 unsigned ConstrainedIntrinsicID,
580 if (CGF.
Builder.getIsFPConstrained())
585 if (CGF.
Builder.getIsFPConstrained())
586 return CGF.
Builder.CreateConstrainedFPCall(F, Args);
588 return CGF.
Builder.CreateCall(F, Args);
594 unsigned IntrinsicID,
595 llvm::StringRef Name =
"") {
599 return CGF.
Builder.CreateCall(F, Src0, Name);
605 unsigned IntrinsicID) {
610 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
616 unsigned IntrinsicID) {
622 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
628 unsigned IntrinsicID) {
633 return CGF.
Builder.CreateCall(F, {Src0, Src1});
639 unsigned IntrinsicID,
640 unsigned ConstrainedIntrinsicID) {
644 if (CGF.
Builder.getIsFPConstrained()) {
645 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
647 {ResultType, Src0->getType()});
648 return CGF.
Builder.CreateConstrainedFPCall(F, {Src0});
652 return CGF.
Builder.CreateCall(F, Src0);
657 llvm::Intrinsic::ID IntrinsicID) {
665 llvm::Value *Call = CGF.
Builder.CreateCall(F, Src0);
667 llvm::Value *Exp = CGF.
Builder.CreateExtractValue(Call, 1);
671 return CGF.
Builder.CreateExtractValue(Call, 0);
677 llvm::CallInst *Call = CGF.
Builder.CreateCall(F,
V);
678 Call->setDoesNotAccessMemory();
687 llvm::Type *Ty =
V->getType();
688 int Width = Ty->getPrimitiveSizeInBits();
689 llvm::Type *IntTy = llvm::IntegerType::get(
C, Width);
691 if (Ty->isPPC_FP128Ty()) {
701 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
706 IntTy = llvm::IntegerType::get(
C, Width);
709 Value *Zero = llvm::Constant::getNullValue(IntTy);
710 return CGF.
Builder.CreateICmpSLT(
V, Zero);
714 const CallExpr *E, llvm::Constant *calleeValue) {
729 const llvm::Intrinsic::ID IntrinsicID,
730 llvm::Value *
X, llvm::Value *Y,
731 llvm::Value *&Carry) {
733 assert(
X->getType() == Y->getType() &&
734 "Arguments must be the same type. (Did you forget to make sure both "
735 "arguments have the same integer width?)");
738 llvm::Value *Tmp = CGF.
Builder.CreateCall(Callee, {
X, Y});
739 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
740 return CGF.
Builder.CreateExtractValue(Tmp, 0);
744 unsigned IntrinsicID,
747 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
749 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
750 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
751 Call->setMetadata(llvm::LLVMContext::MD_noundef,
757 struct WidthAndSignedness {
763static WidthAndSignedness
777static struct WidthAndSignedness
779 assert(Types.size() > 0 &&
"Empty list of types.");
783 for (
const auto &
Type : Types) {
792 for (
const auto &
Type : Types) {
794 if (Width < MinWidth) {
804 if (ArgValue->
getType() != DestType)
806 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
808 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
818 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
823 return ConstantInt::get(ResType, (
Type & 2) ? 0 : -1,
true);
827CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
828 llvm::IntegerType *ResType,
829 llvm::Value *EmittedE,
833 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE, IsDynamic);
834 return ConstantInt::get(ResType, ObjectSize,
true);
847CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
848 llvm::IntegerType *ResType,
849 llvm::Value *EmittedE,
bool IsDynamic) {
853 auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
854 auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
855 if (Param !=
nullptr && PS !=
nullptr &&
857 auto Iter = SizeArguments.find(Param);
858 assert(Iter != SizeArguments.end());
861 auto DIter = LocalDeclMap.find(D);
862 assert(DIter != LocalDeclMap.end());
876 assert(Ptr->
getType()->isPointerTy() &&
877 "Non-pointer passed to __builtin_object_size?");
883 Value *Min = Builder.getInt1((
Type & 2) != 0);
885 Value *NullIsUnknown = Builder.getTrue();
887 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown,
Dynamic});
893 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
894 enum InterlockingKind : uint8_t {
903 InterlockingKind Interlocking;
906 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
910BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
913 case Builtin::BI_bittest:
914 return {TestOnly, Unlocked,
false};
915 case Builtin::BI_bittestandcomplement:
916 return {Complement, Unlocked,
false};
917 case Builtin::BI_bittestandreset:
918 return {Reset, Unlocked,
false};
919 case Builtin::BI_bittestandset:
920 return {Set, Unlocked,
false};
921 case Builtin::BI_interlockedbittestandreset:
922 return {Reset, Sequential,
false};
923 case Builtin::BI_interlockedbittestandset:
924 return {Set, Sequential,
false};
927 case Builtin::BI_bittest64:
928 return {TestOnly, Unlocked,
true};
929 case Builtin::BI_bittestandcomplement64:
930 return {Complement, Unlocked,
true};
931 case Builtin::BI_bittestandreset64:
932 return {Reset, Unlocked,
true};
933 case Builtin::BI_bittestandset64:
934 return {Set, Unlocked,
true};
935 case Builtin::BI_interlockedbittestandreset64:
936 return {Reset, Sequential,
true};
937 case Builtin::BI_interlockedbittestandset64:
938 return {Set, Sequential,
true};
941 case Builtin::BI_interlockedbittestandset_acq:
942 return {Set, Acquire,
false};
943 case Builtin::BI_interlockedbittestandset_rel:
944 return {Set, Release,
false};
945 case Builtin::BI_interlockedbittestandset_nf:
946 return {Set, NoFence,
false};
947 case Builtin::BI_interlockedbittestandreset_acq:
948 return {Reset, Acquire,
false};
949 case Builtin::BI_interlockedbittestandreset_rel:
950 return {Reset, Release,
false};
951 case Builtin::BI_interlockedbittestandreset_nf:
952 return {Reset, NoFence,
false};
954 llvm_unreachable(
"expected only bittest intrinsics");
959 case BitTest::TestOnly:
return '\0';
960 case BitTest::Complement:
return 'c';
961 case BitTest::Reset:
return 'r';
962 case BitTest::Set:
return 's';
964 llvm_unreachable(
"invalid action");
972 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
976 raw_svector_ostream AsmOS(
Asm);
977 if (BT.Interlocking != BitTest::Unlocked)
982 AsmOS << SizeSuffix <<
" $2, ($1)";
985 std::string Constraints =
"={@ccc},r,r,~{cc},~{memory}";
987 if (!MachineClobbers.empty()) {
989 Constraints += MachineClobbers;
991 llvm::IntegerType *IntType = llvm::IntegerType::get(
994 llvm::Type *PtrType = llvm::PointerType::getUnqual(CGF.
getLLVMContext());
995 llvm::FunctionType *FTy =
996 llvm::FunctionType::get(CGF.
Int8Ty, {PtrType, IntType},
false);
998 llvm::InlineAsm *IA =
999 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1000 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
1003static llvm::AtomicOrdering
1006 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
1007 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
1008 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
1009 case BitTest::Release:
return llvm::AtomicOrdering::Release;
1010 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
1012 llvm_unreachable(
"invalid interlocking");
1025 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
1037 BitPos, llvm::ConstantInt::get(BitPos->
getType(), 3),
"bittest.byteidx");
1040 ByteIndex,
"bittest.byteaddr"),
1044 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
1047 Value *Mask =
nullptr;
1048 if (BT.Action != BitTest::TestOnly) {
1049 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
1056 Value *OldByte =
nullptr;
1057 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
1060 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
1061 if (BT.Action == BitTest::Reset) {
1062 Mask = CGF.
Builder.CreateNot(Mask);
1063 RMWOp = llvm::AtomicRMWInst::And;
1070 Value *NewByte =
nullptr;
1071 switch (BT.Action) {
1072 case BitTest::TestOnly:
1075 case BitTest::Complement:
1076 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
1078 case BitTest::Reset:
1079 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
1082 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
1091 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
1093 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
1102 raw_svector_ostream AsmOS(
Asm);
1103 llvm::IntegerType *RetType = CGF.
Int32Ty;
1105 switch (BuiltinID) {
1106 case clang::PPC::BI__builtin_ppc_ldarx:
1110 case clang::PPC::BI__builtin_ppc_lwarx:
1114 case clang::PPC::BI__builtin_ppc_lharx:
1118 case clang::PPC::BI__builtin_ppc_lbarx:
1123 llvm_unreachable(
"Expected only PowerPC load reserve intrinsics");
1126 AsmOS <<
"$0, ${1:y}";
1128 std::string Constraints =
"=r,*Z,~{memory}";
1130 if (!MachineClobbers.empty()) {
1132 Constraints += MachineClobbers;
1135 llvm::Type *PtrType = llvm::PointerType::getUnqual(CGF.
getLLVMContext());
1136 llvm::FunctionType *FTy = llvm::FunctionType::get(RetType, {PtrType},
false);
1138 llvm::InlineAsm *IA =
1139 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1140 llvm::CallInst *CI = CGF.
Builder.CreateCall(IA, {Addr});
1142 0, Attribute::get(CGF.
getLLVMContext(), Attribute::ElementType, RetType));
1147enum class MSVCSetJmpKind {
1159 llvm::Value *Arg1 =
nullptr;
1160 llvm::Type *Arg1Ty =
nullptr;
1162 bool IsVarArg =
false;
1163 if (SJKind == MSVCSetJmpKind::_setjmp3) {
1166 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
1169 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
1172 Arg1 = CGF.
Builder.CreateCall(
1175 Arg1 = CGF.
Builder.CreateCall(
1177 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
1181 llvm::Type *ArgTypes[2] = {CGF.
Int8PtrTy, Arg1Ty};
1182 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
1184 llvm::Attribute::ReturnsTwice);
1186 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
1187 ReturnsTwiceAttr,
true);
1189 llvm::Value *Buf = CGF.
Builder.CreateBitOrPointerCast(
1191 llvm::Value *Args[] = {Buf, Arg1};
1193 CB->setAttributes(ReturnsTwiceAttr);
1241static std::optional<CodeGenFunction::MSVCIntrin>
1244 switch (BuiltinID) {
1246 return std::nullopt;
1247 case clang::ARM::BI_BitScanForward:
1248 case clang::ARM::BI_BitScanForward64:
1249 return MSVCIntrin::_BitScanForward;
1250 case clang::ARM::BI_BitScanReverse:
1251 case clang::ARM::BI_BitScanReverse64:
1252 return MSVCIntrin::_BitScanReverse;
1253 case clang::ARM::BI_InterlockedAnd64:
1254 return MSVCIntrin::_InterlockedAnd;
1255 case clang::ARM::BI_InterlockedExchange64:
1256 return MSVCIntrin::_InterlockedExchange;
1257 case clang::ARM::BI_InterlockedExchangeAdd64:
1258 return MSVCIntrin::_InterlockedExchangeAdd;
1259 case clang::ARM::BI_InterlockedExchangeSub64:
1260 return MSVCIntrin::_InterlockedExchangeSub;
1261 case clang::ARM::BI_InterlockedOr64:
1262 return MSVCIntrin::_InterlockedOr;
1263 case clang::ARM::BI_InterlockedXor64:
1264 return MSVCIntrin::_InterlockedXor;
1265 case clang::ARM::BI_InterlockedDecrement64:
1266 return MSVCIntrin::_InterlockedDecrement;
1267 case clang::ARM::BI_InterlockedIncrement64:
1268 return MSVCIntrin::_InterlockedIncrement;
1269 case clang::ARM::BI_InterlockedExchangeAdd8_acq:
1270 case clang::ARM::BI_InterlockedExchangeAdd16_acq:
1271 case clang::ARM::BI_InterlockedExchangeAdd_acq:
1272 case clang::ARM::BI_InterlockedExchangeAdd64_acq:
1273 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1274 case clang::ARM::BI_InterlockedExchangeAdd8_rel:
1275 case clang::ARM::BI_InterlockedExchangeAdd16_rel:
1276 case clang::ARM::BI_InterlockedExchangeAdd_rel:
1277 case clang::ARM::BI_InterlockedExchangeAdd64_rel:
1278 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1279 case clang::ARM::BI_InterlockedExchangeAdd8_nf:
1280 case clang::ARM::BI_InterlockedExchangeAdd16_nf:
1281 case clang::ARM::BI_InterlockedExchangeAdd_nf:
1282 case clang::ARM::BI_InterlockedExchangeAdd64_nf:
1283 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1284 case clang::ARM::BI_InterlockedExchange8_acq:
1285 case clang::ARM::BI_InterlockedExchange16_acq:
1286 case clang::ARM::BI_InterlockedExchange_acq:
1287 case clang::ARM::BI_InterlockedExchange64_acq:
1288 return MSVCIntrin::_InterlockedExchange_acq;
1289 case clang::ARM::BI_InterlockedExchange8_rel:
1290 case clang::ARM::BI_InterlockedExchange16_rel:
1291 case clang::ARM::BI_InterlockedExchange_rel:
1292 case clang::ARM::BI_InterlockedExchange64_rel:
1293 return MSVCIntrin::_InterlockedExchange_rel;
1294 case clang::ARM::BI_InterlockedExchange8_nf:
1295 case clang::ARM::BI_InterlockedExchange16_nf:
1296 case clang::ARM::BI_InterlockedExchange_nf:
1297 case clang::ARM::BI_InterlockedExchange64_nf:
1298 return MSVCIntrin::_InterlockedExchange_nf;
1299 case clang::ARM::BI_InterlockedCompareExchange8_acq:
1300 case clang::ARM::BI_InterlockedCompareExchange16_acq:
1301 case clang::ARM::BI_InterlockedCompareExchange_acq:
1302 case clang::ARM::BI_InterlockedCompareExchange64_acq:
1303 return MSVCIntrin::_InterlockedCompareExchange_acq;
1304 case clang::ARM::BI_InterlockedCompareExchange8_rel:
1305 case clang::ARM::BI_InterlockedCompareExchange16_rel:
1306 case clang::ARM::BI_InterlockedCompareExchange_rel:
1307 case clang::ARM::BI_InterlockedCompareExchange64_rel:
1308 return MSVCIntrin::_InterlockedCompareExchange_rel;
1309 case clang::ARM::BI_InterlockedCompareExchange8_nf:
1310 case clang::ARM::BI_InterlockedCompareExchange16_nf:
1311 case clang::ARM::BI_InterlockedCompareExchange_nf:
1312 case clang::ARM::BI_InterlockedCompareExchange64_nf:
1313 return MSVCIntrin::_InterlockedCompareExchange_nf;
1314 case clang::ARM::BI_InterlockedOr8_acq:
1315 case clang::ARM::BI_InterlockedOr16_acq:
1316 case clang::ARM::BI_InterlockedOr_acq:
1317 case clang::ARM::BI_InterlockedOr64_acq:
1318 return MSVCIntrin::_InterlockedOr_acq;
1319 case clang::ARM::BI_InterlockedOr8_rel:
1320 case clang::ARM::BI_InterlockedOr16_rel:
1321 case clang::ARM::BI_InterlockedOr_rel:
1322 case clang::ARM::BI_InterlockedOr64_rel:
1323 return MSVCIntrin::_InterlockedOr_rel;
1324 case clang::ARM::BI_InterlockedOr8_nf:
1325 case clang::ARM::BI_InterlockedOr16_nf:
1326 case clang::ARM::BI_InterlockedOr_nf:
1327 case clang::ARM::BI_InterlockedOr64_nf:
1328 return MSVCIntrin::_InterlockedOr_nf;
1329 case clang::ARM::BI_InterlockedXor8_acq:
1330 case clang::ARM::BI_InterlockedXor16_acq:
1331 case clang::ARM::BI_InterlockedXor_acq:
1332 case clang::ARM::BI_InterlockedXor64_acq:
1333 return MSVCIntrin::_InterlockedXor_acq;
1334 case clang::ARM::BI_InterlockedXor8_rel:
1335 case clang::ARM::BI_InterlockedXor16_rel:
1336 case clang::ARM::BI_InterlockedXor_rel:
1337 case clang::ARM::BI_InterlockedXor64_rel:
1338 return MSVCIntrin::_InterlockedXor_rel;
1339 case clang::ARM::BI_InterlockedXor8_nf:
1340 case clang::ARM::BI_InterlockedXor16_nf:
1341 case clang::ARM::BI_InterlockedXor_nf:
1342 case clang::ARM::BI_InterlockedXor64_nf:
1343 return MSVCIntrin::_InterlockedXor_nf;
1344 case clang::ARM::BI_InterlockedAnd8_acq:
1345 case clang::ARM::BI_InterlockedAnd16_acq:
1346 case clang::ARM::BI_InterlockedAnd_acq:
1347 case clang::ARM::BI_InterlockedAnd64_acq:
1348 return MSVCIntrin::_InterlockedAnd_acq;
1349 case clang::ARM::BI_InterlockedAnd8_rel:
1350 case clang::ARM::BI_InterlockedAnd16_rel:
1351 case clang::ARM::BI_InterlockedAnd_rel:
1352 case clang::ARM::BI_InterlockedAnd64_rel:
1353 return MSVCIntrin::_InterlockedAnd_rel;
1354 case clang::ARM::BI_InterlockedAnd8_nf:
1355 case clang::ARM::BI_InterlockedAnd16_nf:
1356 case clang::ARM::BI_InterlockedAnd_nf:
1357 case clang::ARM::BI_InterlockedAnd64_nf:
1358 return MSVCIntrin::_InterlockedAnd_nf;
1359 case clang::ARM::BI_InterlockedIncrement16_acq:
1360 case clang::ARM::BI_InterlockedIncrement_acq:
1361 case clang::ARM::BI_InterlockedIncrement64_acq:
1362 return MSVCIntrin::_InterlockedIncrement_acq;
1363 case clang::ARM::BI_InterlockedIncrement16_rel:
1364 case clang::ARM::BI_InterlockedIncrement_rel:
1365 case clang::ARM::BI_InterlockedIncrement64_rel:
1366 return MSVCIntrin::_InterlockedIncrement_rel;
1367 case clang::ARM::BI_InterlockedIncrement16_nf:
1368 case clang::ARM::BI_InterlockedIncrement_nf:
1369 case clang::ARM::BI_InterlockedIncrement64_nf:
1370 return MSVCIntrin::_InterlockedIncrement_nf;
1371 case clang::ARM::BI_InterlockedDecrement16_acq:
1372 case clang::ARM::BI_InterlockedDecrement_acq:
1373 case clang::ARM::BI_InterlockedDecrement64_acq:
1374 return MSVCIntrin::_InterlockedDecrement_acq;
1375 case clang::ARM::BI_InterlockedDecrement16_rel:
1376 case clang::ARM::BI_InterlockedDecrement_rel:
1377 case clang::ARM::BI_InterlockedDecrement64_rel:
1378 return MSVCIntrin::_InterlockedDecrement_rel;
1379 case clang::ARM::BI_InterlockedDecrement16_nf:
1380 case clang::ARM::BI_InterlockedDecrement_nf:
1381 case clang::ARM::BI_InterlockedDecrement64_nf:
1382 return MSVCIntrin::_InterlockedDecrement_nf;
1384 llvm_unreachable(
"must return from switch");
1387static std::optional<CodeGenFunction::MSVCIntrin>
1390 switch (BuiltinID) {
1392 return std::nullopt;
1393 case clang::AArch64::BI_BitScanForward:
1394 case clang::AArch64::BI_BitScanForward64:
1395 return MSVCIntrin::_BitScanForward;
1396 case clang::AArch64::BI_BitScanReverse:
1397 case clang::AArch64::BI_BitScanReverse64:
1398 return MSVCIntrin::_BitScanReverse;
1399 case clang::AArch64::BI_InterlockedAnd64:
1400 return MSVCIntrin::_InterlockedAnd;
1401 case clang::AArch64::BI_InterlockedExchange64:
1402 return MSVCIntrin::_InterlockedExchange;
1403 case clang::AArch64::BI_InterlockedExchangeAdd64:
1404 return MSVCIntrin::_InterlockedExchangeAdd;
1405 case clang::AArch64::BI_InterlockedExchangeSub64:
1406 return MSVCIntrin::_InterlockedExchangeSub;
1407 case clang::AArch64::BI_InterlockedOr64:
1408 return MSVCIntrin::_InterlockedOr;
1409 case clang::AArch64::BI_InterlockedXor64:
1410 return MSVCIntrin::_InterlockedXor;
1411 case clang::AArch64::BI_InterlockedDecrement64:
1412 return MSVCIntrin::_InterlockedDecrement;
1413 case clang::AArch64::BI_InterlockedIncrement64:
1414 return MSVCIntrin::_InterlockedIncrement;
1415 case clang::AArch64::BI_InterlockedExchangeAdd8_acq:
1416 case clang::AArch64::BI_InterlockedExchangeAdd16_acq:
1417 case clang::AArch64::BI_InterlockedExchangeAdd_acq:
1418 case clang::AArch64::BI_InterlockedExchangeAdd64_acq:
1419 return MSVCIntrin::_InterlockedExchangeAdd_acq;
1420 case clang::AArch64::BI_InterlockedExchangeAdd8_rel:
1421 case clang::AArch64::BI_InterlockedExchangeAdd16_rel:
1422 case clang::AArch64::BI_InterlockedExchangeAdd_rel:
1423 case clang::AArch64::BI_InterlockedExchangeAdd64_rel:
1424 return MSVCIntrin::_InterlockedExchangeAdd_rel;
1425 case clang::AArch64::BI_InterlockedExchangeAdd8_nf:
1426 case clang::AArch64::BI_InterlockedExchangeAdd16_nf:
1427 case clang::AArch64::BI_InterlockedExchangeAdd_nf:
1428 case clang::AArch64::BI_InterlockedExchangeAdd64_nf:
1429 return MSVCIntrin::_InterlockedExchangeAdd_nf;
1430 case clang::AArch64::BI_InterlockedExchange8_acq:
1431 case clang::AArch64::BI_InterlockedExchange16_acq:
1432 case clang::AArch64::BI_InterlockedExchange_acq:
1433 case clang::AArch64::BI_InterlockedExchange64_acq:
1434 return MSVCIntrin::_InterlockedExchange_acq;
1435 case clang::AArch64::BI_InterlockedExchange8_rel:
1436 case clang::AArch64::BI_InterlockedExchange16_rel:
1437 case clang::AArch64::BI_InterlockedExchange_rel:
1438 case clang::AArch64::BI_InterlockedExchange64_rel:
1439 return MSVCIntrin::_InterlockedExchange_rel;
1440 case clang::AArch64::BI_InterlockedExchange8_nf:
1441 case clang::AArch64::BI_InterlockedExchange16_nf:
1442 case clang::AArch64::BI_InterlockedExchange_nf:
1443 case clang::AArch64::BI_InterlockedExchange64_nf:
1444 return MSVCIntrin::_InterlockedExchange_nf;
1445 case clang::AArch64::BI_InterlockedCompareExchange8_acq:
1446 case clang::AArch64::BI_InterlockedCompareExchange16_acq:
1447 case clang::AArch64::BI_InterlockedCompareExchange_acq:
1448 case clang::AArch64::BI_InterlockedCompareExchange64_acq:
1449 return MSVCIntrin::_InterlockedCompareExchange_acq;
1450 case clang::AArch64::BI_InterlockedCompareExchange8_rel:
1451 case clang::AArch64::BI_InterlockedCompareExchange16_rel:
1452 case clang::AArch64::BI_InterlockedCompareExchange_rel:
1453 case clang::AArch64::BI_InterlockedCompareExchange64_rel:
1454 return MSVCIntrin::_InterlockedCompareExchange_rel;
1455 case clang::AArch64::BI_InterlockedCompareExchange8_nf:
1456 case clang::AArch64::BI_InterlockedCompareExchange16_nf:
1457 case clang::AArch64::BI_InterlockedCompareExchange_nf:
1458 case clang::AArch64::BI_InterlockedCompareExchange64_nf:
1459 return MSVCIntrin::_InterlockedCompareExchange_nf;
1460 case clang::AArch64::BI_InterlockedCompareExchange128:
1461 return MSVCIntrin::_InterlockedCompareExchange128;
1462 case clang::AArch64::BI_InterlockedCompareExchange128_acq:
1463 return MSVCIntrin::_InterlockedCompareExchange128_acq;
1464 case clang::AArch64::BI_InterlockedCompareExchange128_nf:
1465 return MSVCIntrin::_InterlockedCompareExchange128_nf;
1466 case clang::AArch64::BI_InterlockedCompareExchange128_rel:
1467 return MSVCIntrin::_InterlockedCompareExchange128_rel;
1468 case clang::AArch64::BI_InterlockedOr8_acq:
1469 case clang::AArch64::BI_InterlockedOr16_acq:
1470 case clang::AArch64::BI_InterlockedOr_acq:
1471 case clang::AArch64::BI_InterlockedOr64_acq:
1472 return MSVCIntrin::_InterlockedOr_acq;
1473 case clang::AArch64::BI_InterlockedOr8_rel:
1474 case clang::AArch64::BI_InterlockedOr16_rel:
1475 case clang::AArch64::BI_InterlockedOr_rel:
1476 case clang::AArch64::BI_InterlockedOr64_rel:
1477 return MSVCIntrin::_InterlockedOr_rel;
1478 case clang::AArch64::BI_InterlockedOr8_nf:
1479 case clang::AArch64::BI_InterlockedOr16_nf:
1480 case clang::AArch64::BI_InterlockedOr_nf:
1481 case clang::AArch64::BI_InterlockedOr64_nf:
1482 return MSVCIntrin::_InterlockedOr_nf;
1483 case clang::AArch64::BI_InterlockedXor8_acq:
1484 case clang::AArch64::BI_InterlockedXor16_acq:
1485 case clang::AArch64::BI_InterlockedXor_acq:
1486 case clang::AArch64::BI_InterlockedXor64_acq:
1487 return MSVCIntrin::_InterlockedXor_acq;
1488 case clang::AArch64::BI_InterlockedXor8_rel:
1489 case clang::AArch64::BI_InterlockedXor16_rel:
1490 case clang::AArch64::BI_InterlockedXor_rel:
1491 case clang::AArch64::BI_InterlockedXor64_rel:
1492 return MSVCIntrin::_InterlockedXor_rel;
1493 case clang::AArch64::BI_InterlockedXor8_nf:
1494 case clang::AArch64::BI_InterlockedXor16_nf:
1495 case clang::AArch64::BI_InterlockedXor_nf:
1496 case clang::AArch64::BI_InterlockedXor64_nf:
1497 return MSVCIntrin::_InterlockedXor_nf;
1498 case clang::AArch64::BI_InterlockedAnd8_acq:
1499 case clang::AArch64::BI_InterlockedAnd16_acq:
1500 case clang::AArch64::BI_InterlockedAnd_acq:
1501 case clang::AArch64::BI_InterlockedAnd64_acq:
1502 return MSVCIntrin::_InterlockedAnd_acq;
1503 case clang::AArch64::BI_InterlockedAnd8_rel:
1504 case clang::AArch64::BI_InterlockedAnd16_rel:
1505 case clang::AArch64::BI_InterlockedAnd_rel:
1506 case clang::AArch64::BI_InterlockedAnd64_rel:
1507 return MSVCIntrin::_InterlockedAnd_rel;
1508 case clang::AArch64::BI_InterlockedAnd8_nf:
1509 case clang::AArch64::BI_InterlockedAnd16_nf:
1510 case clang::AArch64::BI_InterlockedAnd_nf:
1511 case clang::AArch64::BI_InterlockedAnd64_nf:
1512 return MSVCIntrin::_InterlockedAnd_nf;
1513 case clang::AArch64::BI_InterlockedIncrement16_acq:
1514 case clang::AArch64::BI_InterlockedIncrement_acq:
1515 case clang::AArch64::BI_InterlockedIncrement64_acq:
1516 return MSVCIntrin::_InterlockedIncrement_acq;
1517 case clang::AArch64::BI_InterlockedIncrement16_rel:
1518 case clang::AArch64::BI_InterlockedIncrement_rel:
1519 case clang::AArch64::BI_InterlockedIncrement64_rel:
1520 return MSVCIntrin::_InterlockedIncrement_rel;
1521 case clang::AArch64::BI_InterlockedIncrement16_nf:
1522 case clang::AArch64::BI_InterlockedIncrement_nf:
1523 case clang::AArch64::BI_InterlockedIncrement64_nf:
1524 return MSVCIntrin::_InterlockedIncrement_nf;
1525 case clang::AArch64::BI_InterlockedDecrement16_acq:
1526 case clang::AArch64::BI_InterlockedDecrement_acq:
1527 case clang::AArch64::BI_InterlockedDecrement64_acq:
1528 return MSVCIntrin::_InterlockedDecrement_acq;
1529 case clang::AArch64::BI_InterlockedDecrement16_rel:
1530 case clang::AArch64::BI_InterlockedDecrement_rel:
1531 case clang::AArch64::BI_InterlockedDecrement64_rel:
1532 return MSVCIntrin::_InterlockedDecrement_rel;
1533 case clang::AArch64::BI_InterlockedDecrement16_nf:
1534 case clang::AArch64::BI_InterlockedDecrement_nf:
1535 case clang::AArch64::BI_InterlockedDecrement64_nf:
1536 return MSVCIntrin::_InterlockedDecrement_nf;
1538 llvm_unreachable(
"must return from switch");
1541static std::optional<CodeGenFunction::MSVCIntrin>
1544 switch (BuiltinID) {
1546 return std::nullopt;
1547 case clang::X86::BI_BitScanForward:
1548 case clang::X86::BI_BitScanForward64:
1549 return MSVCIntrin::_BitScanForward;
1550 case clang::X86::BI_BitScanReverse:
1551 case clang::X86::BI_BitScanReverse64:
1552 return MSVCIntrin::_BitScanReverse;
1553 case clang::X86::BI_InterlockedAnd64:
1554 return MSVCIntrin::_InterlockedAnd;
1555 case clang::X86::BI_InterlockedCompareExchange128:
1556 return MSVCIntrin::_InterlockedCompareExchange128;
1557 case clang::X86::BI_InterlockedExchange64:
1558 return MSVCIntrin::_InterlockedExchange;
1559 case clang::X86::BI_InterlockedExchangeAdd64:
1560 return MSVCIntrin::_InterlockedExchangeAdd;
1561 case clang::X86::BI_InterlockedExchangeSub64:
1562 return MSVCIntrin::_InterlockedExchangeSub;
1563 case clang::X86::BI_InterlockedOr64:
1564 return MSVCIntrin::_InterlockedOr;
1565 case clang::X86::BI_InterlockedXor64:
1566 return MSVCIntrin::_InterlockedXor;
1567 case clang::X86::BI_InterlockedDecrement64:
1568 return MSVCIntrin::_InterlockedDecrement;
1569 case clang::X86::BI_InterlockedIncrement64:
1570 return MSVCIntrin::_InterlockedIncrement;
1572 llvm_unreachable(
"must return from switch");
1578 switch (BuiltinID) {
1579 case MSVCIntrin::_BitScanForward:
1580 case MSVCIntrin::_BitScanReverse: {
1584 llvm::Type *ArgType = ArgValue->
getType();
1585 llvm::Type *IndexType = IndexAddress.getElementType();
1588 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1589 Value *ResZero = llvm::Constant::getNullValue(ResultType);
1590 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1592 BasicBlock *
Begin = Builder.GetInsertBlock();
1594 Builder.SetInsertPoint(End);
1595 PHINode *
Result = Builder.CreatePHI(ResultType, 2,
"bitscan_result");
1597 Builder.SetInsertPoint(
Begin);
1598 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1600 Builder.CreateCondBr(IsZero, End, NotZero);
1603 Builder.SetInsertPoint(NotZero);
1605 if (BuiltinID == MSVCIntrin::_BitScanForward) {
1607 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1608 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
1609 Builder.CreateStore(ZeroCount, IndexAddress,
false);
1612 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1615 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1616 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
1617 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1618 Builder.CreateStore(Index, IndexAddress,
false);
1620 Builder.CreateBr(End);
1621 Result->addIncoming(ResOne, NotZero);
1623 Builder.SetInsertPoint(End);
1626 case MSVCIntrin::_InterlockedAnd:
1628 case MSVCIntrin::_InterlockedExchange:
1630 case MSVCIntrin::_InterlockedExchangeAdd:
1632 case MSVCIntrin::_InterlockedExchangeSub:
1634 case MSVCIntrin::_InterlockedOr:
1636 case MSVCIntrin::_InterlockedXor:
1638 case MSVCIntrin::_InterlockedExchangeAdd_acq:
1640 AtomicOrdering::Acquire);
1641 case MSVCIntrin::_InterlockedExchangeAdd_rel:
1643 AtomicOrdering::Release);
1644 case MSVCIntrin::_InterlockedExchangeAdd_nf:
1646 AtomicOrdering::Monotonic);
1647 case MSVCIntrin::_InterlockedExchange_acq:
1649 AtomicOrdering::Acquire);
1650 case MSVCIntrin::_InterlockedExchange_rel:
1652 AtomicOrdering::Release);
1653 case MSVCIntrin::_InterlockedExchange_nf:
1655 AtomicOrdering::Monotonic);
1656 case MSVCIntrin::_InterlockedCompareExchange_acq:
1658 case MSVCIntrin::_InterlockedCompareExchange_rel:
1660 case MSVCIntrin::_InterlockedCompareExchange_nf:
1662 case MSVCIntrin::_InterlockedCompareExchange128:
1664 *
this, E, AtomicOrdering::SequentiallyConsistent);
1665 case MSVCIntrin::_InterlockedCompareExchange128_acq:
1667 case MSVCIntrin::_InterlockedCompareExchange128_rel:
1669 case MSVCIntrin::_InterlockedCompareExchange128_nf:
1671 case MSVCIntrin::_InterlockedOr_acq:
1673 AtomicOrdering::Acquire);
1674 case MSVCIntrin::_InterlockedOr_rel:
1676 AtomicOrdering::Release);
1677 case MSVCIntrin::_InterlockedOr_nf:
1679 AtomicOrdering::Monotonic);
1680 case MSVCIntrin::_InterlockedXor_acq:
1682 AtomicOrdering::Acquire);
1683 case MSVCIntrin::_InterlockedXor_rel:
1685 AtomicOrdering::Release);
1686 case MSVCIntrin::_InterlockedXor_nf:
1688 AtomicOrdering::Monotonic);
1689 case MSVCIntrin::_InterlockedAnd_acq:
1691 AtomicOrdering::Acquire);
1692 case MSVCIntrin::_InterlockedAnd_rel:
1694 AtomicOrdering::Release);
1695 case MSVCIntrin::_InterlockedAnd_nf:
1697 AtomicOrdering::Monotonic);
1698 case MSVCIntrin::_InterlockedIncrement_acq:
1700 case MSVCIntrin::_InterlockedIncrement_rel:
1702 case MSVCIntrin::_InterlockedIncrement_nf:
1704 case MSVCIntrin::_InterlockedDecrement_acq:
1706 case MSVCIntrin::_InterlockedDecrement_rel:
1708 case MSVCIntrin::_InterlockedDecrement_nf:
1711 case MSVCIntrin::_InterlockedDecrement:
1713 case MSVCIntrin::_InterlockedIncrement:
1716 case MSVCIntrin::__fastfail: {
1721 StringRef
Asm, Constraints;
1726 case llvm::Triple::x86:
1727 case llvm::Triple::x86_64:
1729 Constraints =
"{cx}";
1731 case llvm::Triple::thumb:
1733 Constraints =
"{r0}";
1735 case llvm::Triple::aarch64:
1736 Asm =
"brk #0xF003";
1737 Constraints =
"{w0}";
1739 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy, {
Int32Ty},
false);
1740 llvm::InlineAsm *IA =
1741 llvm::InlineAsm::get(FTy,
Asm, Constraints,
true);
1742 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1744 llvm::Attribute::NoReturn);
1746 CI->setAttributes(NoReturnAttr);
1750 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1756 CallObjCArcUse(llvm::Value *
object) :
object(
object) {}
1766 BuiltinCheckKind Kind) {
1768 &&
"Unsupported builtin check kind");
1774 SanitizerScope SanScope(
this);
1776 ArgValue, llvm::Constant::getNullValue(ArgValue->
getType()));
1778 SanitizerHandler::InvalidBuiltin,
1780 llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
1787 QualType UnsignedTy =
C.getIntTypeForBitwidth(Size * 8,
false);
1798 raw_svector_ostream OS(Name);
1799 OS <<
"__os_log_helper";
1803 for (
const auto &Item : Layout.
Items)
1804 OS <<
"_" <<
int(Item.getSizeByte()) <<
"_"
1805 <<
int(Item.getDescriptorByte());
1808 if (llvm::Function *F =
CGM.
getModule().getFunction(Name))
1818 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
1819 char Size = Layout.
Items[I].getSizeByte();
1826 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
1828 ArgTys.emplace_back(ArgTy);
1839 llvm::Function *Fn = llvm::Function::Create(
1840 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &
CGM.
getModule());
1841 Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1844 Fn->setDoesNotThrow();
1848 Fn->addFnAttr(llvm::Attribute::NoInline);
1861 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"summary"));
1863 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"numArgs"));
1866 for (
const auto &Item : Layout.
Items) {
1867 Builder.CreateStore(
1868 Builder.getInt8(Item.getDescriptorByte()),
1869 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argDescriptor"));
1870 Builder.CreateStore(
1871 Builder.getInt8(Item.getSizeByte()),
1872 Builder.CreateConstByteGEP(BufAddr,
Offset++,
"argSize"));
1875 if (!
Size.getQuantity())
1879 Address Addr = Builder.CreateConstByteGEP(BufAddr,
Offset,
"argData");
1881 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1893 "__builtin_os_log_format takes at least 2 arguments");
1904 for (
const auto &Item : Layout.
Items) {
1905 int Size = Item.getSizeByte();
1909 llvm::Value *ArgVal;
1913 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1914 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
1915 ArgVal = llvm::Constant::getIntegerValue(
Int64Ty, llvm::APInt(64, Val));
1916 }
else if (
const Expr *TheExpr = Item.getExpr()) {
1922 auto LifetimeExtendObject = [&](
const Expr *E) {
1930 if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
1935 if (TheExpr->getType()->isObjCRetainableType() &&
1936 getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
1938 "Only scalar can be a ObjC retainable type");
1939 if (!isa<Constant>(ArgVal)) {
1945 Builder.CreateStore(ArgVal, Addr);
1957 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1960 unsigned ArgValSize =
1964 ArgVal = Builder.CreateBitOrPointerCast(ArgVal,
IntTy);
1967 ArgVal = Builder.CreateZExtOrBitCast(ArgVal,
ConvertType(ArgTy));
1980 unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info,
1981 WidthAndSignedness ResultInfo) {
1982 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1983 Op1Info.Width == Op2Info.Width && Op2Info.Width == ResultInfo.Width &&
1984 !Op1Info.Signed && !Op2Info.Signed && ResultInfo.Signed;
1989 const clang::Expr *Op2, WidthAndSignedness Op2Info,
1991 WidthAndSignedness ResultInfo) {
1993 Builtin::BI__builtin_mul_overflow, Op1Info, Op2Info, ResultInfo) &&
1994 "Cannot specialize this multiply");
1999 llvm::Value *HasOverflow;
2001 CGF, llvm::Intrinsic::umul_with_overflow, V1, V2, HasOverflow);
2006 auto IntMax = llvm::APInt::getSignedMaxValue(ResultInfo.Width);
2007 llvm::Value *IntMaxValue = llvm::ConstantInt::get(
Result->getType(), IntMax);
2009 llvm::Value *IntMaxOverflow = CGF.
Builder.CreateICmpUGT(
Result, IntMaxValue);
2010 HasOverflow = CGF.
Builder.CreateOr(HasOverflow, IntMaxOverflow);
2022 WidthAndSignedness Op1Info,
2023 WidthAndSignedness Op2Info,
2024 WidthAndSignedness ResultInfo) {
2025 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
2026 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
2027 Op1Info.Signed != Op2Info.Signed;
2034 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
2035 WidthAndSignedness Op2Info,
2037 WidthAndSignedness ResultInfo) {
2039 Op2Info, ResultInfo) &&
2040 "Not a mixed-sign multipliction we can specialize");
2043 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
2044 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
2047 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
2048 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
2051 if (SignedOpWidth < UnsignedOpWidth)
2053 if (UnsignedOpWidth < SignedOpWidth)
2056 llvm::Type *OpTy =
Signed->getType();
2057 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
2060 unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
2063 llvm::Value *IsNegative = CGF.
Builder.CreateICmpSLT(
Signed, Zero);
2064 llvm::Value *AbsOfNegative = CGF.
Builder.CreateSub(Zero,
Signed);
2065 llvm::Value *AbsSigned =
2066 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative,
Signed);
2069 llvm::Value *UnsignedOverflow;
2070 llvm::Value *UnsignedResult =
2074 llvm::Value *Overflow, *
Result;
2075 if (ResultInfo.Signed) {
2079 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zext(OpWidth);
2080 llvm::Value *MaxResult =
2081 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
2082 CGF.
Builder.CreateZExt(IsNegative, OpTy));
2083 llvm::Value *SignedOverflow =
2084 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
2085 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
2088 llvm::Value *NegativeResult = CGF.
Builder.CreateNeg(UnsignedResult);
2089 llvm::Value *SignedResult =
2090 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
2094 llvm::Value *Underflow = CGF.
Builder.CreateAnd(
2095 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
2096 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
2097 if (ResultInfo.Width < OpWidth) {
2099 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
2100 llvm::Value *TruncOverflow = CGF.
Builder.CreateICmpUGT(
2101 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
2102 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
2107 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
2111 assert(Overflow &&
Result &&
"Missing overflow or result");
2122 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
2131 if (!Seen.insert(Record).second)
2134 assert(Record->hasDefinition() &&
2135 "Incomplete types should already be diagnosed");
2137 if (Record->isDynamicClass())
2162 llvm::Type *Ty = Src->getType();
2163 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty,
false);
2166 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
2168 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
2173 switch (BuiltinID) {
2174#define MUTATE_LDBL(func) \
2175 case Builtin::BI__builtin_##func##l: \
2176 return Builtin::BI__builtin_##func##f128;
2245 if (CGF.
Builder.getIsFPConstrained() &&
2246 CGF.
Builder.getDefaultConstrainedExcept() != fp::ebIgnore) {
2262 !
Result.hasSideEffects()) {
2266 if (
Result.Val.isFloat())
2275 if (
getTarget().getTriple().isPPC64() &&
2276 &
getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
2283 const unsigned BuiltinIDIfNoAsmLabel =
2284 FD->
hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
2293 bool ConstWithoutErrnoAndExceptions =
2295 bool ConstWithoutExceptions =
2297 if (FD->
hasAttr<ConstAttr>() ||
2298 ((ConstWithoutErrnoAndExceptions || ConstWithoutExceptions) &&
2299 (!ConstWithoutErrnoAndExceptions || (!
getLangOpts().MathErrno)))) {
2300 switch (BuiltinIDIfNoAsmLabel) {
2301 case Builtin::BIceil:
2302 case Builtin::BIceilf:
2303 case Builtin::BIceill:
2304 case Builtin::BI__builtin_ceil:
2305 case Builtin::BI__builtin_ceilf:
2306 case Builtin::BI__builtin_ceilf16:
2307 case Builtin::BI__builtin_ceill:
2308 case Builtin::BI__builtin_ceilf128:
2311 Intrinsic::experimental_constrained_ceil));
2313 case Builtin::BIcopysign:
2314 case Builtin::BIcopysignf:
2315 case Builtin::BIcopysignl:
2316 case Builtin::BI__builtin_copysign:
2317 case Builtin::BI__builtin_copysignf:
2318 case Builtin::BI__builtin_copysignf16:
2319 case Builtin::BI__builtin_copysignl:
2320 case Builtin::BI__builtin_copysignf128:
2323 case Builtin::BIcos:
2324 case Builtin::BIcosf:
2325 case Builtin::BIcosl:
2326 case Builtin::BI__builtin_cos:
2327 case Builtin::BI__builtin_cosf:
2328 case Builtin::BI__builtin_cosf16:
2329 case Builtin::BI__builtin_cosl:
2330 case Builtin::BI__builtin_cosf128:
2333 Intrinsic::experimental_constrained_cos));
2335 case Builtin::BIexp:
2336 case Builtin::BIexpf:
2337 case Builtin::BIexpl:
2338 case Builtin::BI__builtin_exp:
2339 case Builtin::BI__builtin_expf:
2340 case Builtin::BI__builtin_expf16:
2341 case Builtin::BI__builtin_expl:
2342 case Builtin::BI__builtin_expf128:
2345 Intrinsic::experimental_constrained_exp));
2347 case Builtin::BIexp2:
2348 case Builtin::BIexp2f:
2349 case Builtin::BIexp2l:
2350 case Builtin::BI__builtin_exp2:
2351 case Builtin::BI__builtin_exp2f:
2352 case Builtin::BI__builtin_exp2f16:
2353 case Builtin::BI__builtin_exp2l:
2354 case Builtin::BI__builtin_exp2f128:
2357 Intrinsic::experimental_constrained_exp2));
2359 case Builtin::BIfabs:
2360 case Builtin::BIfabsf:
2361 case Builtin::BIfabsl:
2362 case Builtin::BI__builtin_fabs:
2363 case Builtin::BI__builtin_fabsf:
2364 case Builtin::BI__builtin_fabsf16:
2365 case Builtin::BI__builtin_fabsl:
2366 case Builtin::BI__builtin_fabsf128:
2369 case Builtin::BIfloor:
2370 case Builtin::BIfloorf:
2371 case Builtin::BIfloorl:
2372 case Builtin::BI__builtin_floor:
2373 case Builtin::BI__builtin_floorf:
2374 case Builtin::BI__builtin_floorf16:
2375 case Builtin::BI__builtin_floorl:
2376 case Builtin::BI__builtin_floorf128:
2379 Intrinsic::experimental_constrained_floor));
2381 case Builtin::BIfma:
2382 case Builtin::BIfmaf:
2383 case Builtin::BIfmal:
2384 case Builtin::BI__builtin_fma:
2385 case Builtin::BI__builtin_fmaf:
2386 case Builtin::BI__builtin_fmaf16:
2387 case Builtin::BI__builtin_fmal:
2388 case Builtin::BI__builtin_fmaf128:
2391 Intrinsic::experimental_constrained_fma));
2393 case Builtin::BIfmax:
2394 case Builtin::BIfmaxf:
2395 case Builtin::BIfmaxl:
2396 case Builtin::BI__builtin_fmax:
2397 case Builtin::BI__builtin_fmaxf:
2398 case Builtin::BI__builtin_fmaxf16:
2399 case Builtin::BI__builtin_fmaxl:
2400 case Builtin::BI__builtin_fmaxf128:
2403 Intrinsic::experimental_constrained_maxnum));
2405 case Builtin::BIfmin:
2406 case Builtin::BIfminf:
2407 case Builtin::BIfminl:
2408 case Builtin::BI__builtin_fmin:
2409 case Builtin::BI__builtin_fminf:
2410 case Builtin::BI__builtin_fminf16:
2411 case Builtin::BI__builtin_fminl:
2412 case Builtin::BI__builtin_fminf128:
2415 Intrinsic::experimental_constrained_minnum));
2419 case Builtin::BIfmod:
2420 case Builtin::BIfmodf:
2421 case Builtin::BIfmodl:
2422 case Builtin::BI__builtin_fmod:
2423 case Builtin::BI__builtin_fmodf:
2424 case Builtin::BI__builtin_fmodf16:
2425 case Builtin::BI__builtin_fmodl:
2426 case Builtin::BI__builtin_fmodf128: {
2427 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2430 return RValue::get(Builder.CreateFRem(Arg1, Arg2,
"fmod"));
2433 case Builtin::BIlog:
2434 case Builtin::BIlogf:
2435 case Builtin::BIlogl:
2436 case Builtin::BI__builtin_log:
2437 case Builtin::BI__builtin_logf:
2438 case Builtin::BI__builtin_logf16:
2439 case Builtin::BI__builtin_logl:
2440 case Builtin::BI__builtin_logf128:
2443 Intrinsic::experimental_constrained_log));
2445 case Builtin::BIlog10:
2446 case Builtin::BIlog10f:
2447 case Builtin::BIlog10l:
2448 case Builtin::BI__builtin_log10:
2449 case Builtin::BI__builtin_log10f:
2450 case Builtin::BI__builtin_log10f16:
2451 case Builtin::BI__builtin_log10l:
2452 case Builtin::BI__builtin_log10f128:
2455 Intrinsic::experimental_constrained_log10));
2457 case Builtin::BIlog2:
2458 case Builtin::BIlog2f:
2459 case Builtin::BIlog2l:
2460 case Builtin::BI__builtin_log2:
2461 case Builtin::BI__builtin_log2f:
2462 case Builtin::BI__builtin_log2f16:
2463 case Builtin::BI__builtin_log2l:
2464 case Builtin::BI__builtin_log2f128:
2467 Intrinsic::experimental_constrained_log2));
2469 case Builtin::BInearbyint:
2470 case Builtin::BInearbyintf:
2471 case Builtin::BInearbyintl:
2472 case Builtin::BI__builtin_nearbyint:
2473 case Builtin::BI__builtin_nearbyintf:
2474 case Builtin::BI__builtin_nearbyintl:
2475 case Builtin::BI__builtin_nearbyintf128:
2477 Intrinsic::nearbyint,
2478 Intrinsic::experimental_constrained_nearbyint));
2480 case Builtin::BIpow:
2481 case Builtin::BIpowf:
2482 case Builtin::BIpowl:
2483 case Builtin::BI__builtin_pow:
2484 case Builtin::BI__builtin_powf:
2485 case Builtin::BI__builtin_powf16:
2486 case Builtin::BI__builtin_powl:
2487 case Builtin::BI__builtin_powf128:
2490 Intrinsic::experimental_constrained_pow));
2492 case Builtin::BIrint:
2493 case Builtin::BIrintf:
2494 case Builtin::BIrintl:
2495 case Builtin::BI__builtin_rint:
2496 case Builtin::BI__builtin_rintf:
2497 case Builtin::BI__builtin_rintf16:
2498 case Builtin::BI__builtin_rintl:
2499 case Builtin::BI__builtin_rintf128:
2502 Intrinsic::experimental_constrained_rint));
2504 case Builtin::BIround:
2505 case Builtin::BIroundf:
2506 case Builtin::BIroundl:
2507 case Builtin::BI__builtin_round:
2508 case Builtin::BI__builtin_roundf:
2509 case Builtin::BI__builtin_roundf16:
2510 case Builtin::BI__builtin_roundl:
2511 case Builtin::BI__builtin_roundf128:
2514 Intrinsic::experimental_constrained_round));
2516 case Builtin::BIroundeven:
2517 case Builtin::BIroundevenf:
2518 case Builtin::BIroundevenl:
2519 case Builtin::BI__builtin_roundeven:
2520 case Builtin::BI__builtin_roundevenf:
2521 case Builtin::BI__builtin_roundevenf16:
2522 case Builtin::BI__builtin_roundevenl:
2523 case Builtin::BI__builtin_roundevenf128:
2525 Intrinsic::roundeven,
2526 Intrinsic::experimental_constrained_roundeven));
2528 case Builtin::BIsin:
2529 case Builtin::BIsinf:
2530 case Builtin::BIsinl:
2531 case Builtin::BI__builtin_sin:
2532 case Builtin::BI__builtin_sinf:
2533 case Builtin::BI__builtin_sinf16:
2534 case Builtin::BI__builtin_sinl:
2535 case Builtin::BI__builtin_sinf128:
2538 Intrinsic::experimental_constrained_sin));
2540 case Builtin::BIsqrt:
2541 case Builtin::BIsqrtf:
2542 case Builtin::BIsqrtl:
2543 case Builtin::BI__builtin_sqrt:
2544 case Builtin::BI__builtin_sqrtf:
2545 case Builtin::BI__builtin_sqrtf16:
2546 case Builtin::BI__builtin_sqrtl:
2547 case Builtin::BI__builtin_sqrtf128: {
2549 *
this, E, Intrinsic::sqrt, Intrinsic::experimental_constrained_sqrt);
2553 case Builtin::BItrunc:
2554 case Builtin::BItruncf:
2555 case Builtin::BItruncl:
2556 case Builtin::BI__builtin_trunc:
2557 case Builtin::BI__builtin_truncf:
2558 case Builtin::BI__builtin_truncf16:
2559 case Builtin::BI__builtin_truncl:
2560 case Builtin::BI__builtin_truncf128:
2563 Intrinsic::experimental_constrained_trunc));
2565 case Builtin::BIlround:
2566 case Builtin::BIlroundf:
2567 case Builtin::BIlroundl:
2568 case Builtin::BI__builtin_lround:
2569 case Builtin::BI__builtin_lroundf:
2570 case Builtin::BI__builtin_lroundl:
2571 case Builtin::BI__builtin_lroundf128:
2573 *
this, E, Intrinsic::lround,
2574 Intrinsic::experimental_constrained_lround));
2576 case Builtin::BIllround:
2577 case Builtin::BIllroundf:
2578 case Builtin::BIllroundl:
2579 case Builtin::BI__builtin_llround:
2580 case Builtin::BI__builtin_llroundf:
2581 case Builtin::BI__builtin_llroundl:
2582 case Builtin::BI__builtin_llroundf128:
2584 *
this, E, Intrinsic::llround,
2585 Intrinsic::experimental_constrained_llround));
2587 case Builtin::BIlrint:
2588 case Builtin::BIlrintf:
2589 case Builtin::BIlrintl:
2590 case Builtin::BI__builtin_lrint:
2591 case Builtin::BI__builtin_lrintf:
2592 case Builtin::BI__builtin_lrintl:
2593 case Builtin::BI__builtin_lrintf128:
2595 *
this, E, Intrinsic::lrint,
2596 Intrinsic::experimental_constrained_lrint));
2598 case Builtin::BIllrint:
2599 case Builtin::BIllrintf:
2600 case Builtin::BIllrintl:
2601 case Builtin::BI__builtin_llrint:
2602 case Builtin::BI__builtin_llrintf:
2603 case Builtin::BI__builtin_llrintl:
2604 case Builtin::BI__builtin_llrintf128:
2606 *
this, E, Intrinsic::llrint,
2607 Intrinsic::experimental_constrained_llrint));
2608 case Builtin::BI__builtin_ldexp:
2609 case Builtin::BI__builtin_ldexpf:
2610 case Builtin::BI__builtin_ldexpl:
2611 case Builtin::BI__builtin_ldexpf16:
2612 case Builtin::BI__builtin_ldexpf128: {
2614 *
this, E, Intrinsic::ldexp,
2615 Intrinsic::experimental_constrained_ldexp));
2622 switch (BuiltinIDIfNoAsmLabel) {
2624 case Builtin::BI__builtin___CFStringMakeConstantString:
2625 case Builtin::BI__builtin___NSStringMakeConstantString:
2627 case Builtin::BI__builtin_stdarg_start:
2628 case Builtin::BI__builtin_va_start:
2629 case Builtin::BI__va_start:
2630 case Builtin::BI__builtin_va_end:
2634 BuiltinID != Builtin::BI__builtin_va_end);
2636 case Builtin::BI__builtin_va_copy: {
2642 DstPtr = Builder.CreateBitCast(DstPtr,
Type);
2643 SrcPtr = Builder.CreateBitCast(SrcPtr,
Type);
2644 Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::vacopy), {DstPtr, SrcPtr});
2647 case Builtin::BI__builtin_abs:
2648 case Builtin::BI__builtin_labs:
2649 case Builtin::BI__builtin_llabs: {
2653 Value *NegOp = Builder.CreateNSWNeg(ArgValue,
"neg");
2654 Constant *
Zero = llvm::Constant::getNullValue(ArgValue->
getType());
2655 Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
2656 Value *
Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue,
"abs");
2659 case Builtin::BI__builtin_complex: {
2664 case Builtin::BI__builtin_conj:
2665 case Builtin::BI__builtin_conjf:
2666 case Builtin::BI__builtin_conjl:
2667 case Builtin::BIconj:
2668 case Builtin::BIconjf:
2669 case Builtin::BIconjl: {
2671 Value *Real = ComplexVal.first;
2672 Value *Imag = ComplexVal.second;
2673 Imag = Builder.CreateFNeg(Imag,
"neg");
2676 case Builtin::BI__builtin_creal:
2677 case Builtin::BI__builtin_crealf:
2678 case Builtin::BI__builtin_creall:
2679 case Builtin::BIcreal:
2680 case Builtin::BIcrealf:
2681 case Builtin::BIcreall: {
2686 case Builtin::BI__builtin_preserve_access_index: {
2707 case Builtin::BI__builtin_cimag:
2708 case Builtin::BI__builtin_cimagf:
2709 case Builtin::BI__builtin_cimagl:
2710 case Builtin::BIcimag:
2711 case Builtin::BIcimagf:
2712 case Builtin::BIcimagl: {
2717 case Builtin::BI__builtin_clrsb:
2718 case Builtin::BI__builtin_clrsbl:
2719 case Builtin::BI__builtin_clrsbll: {
2723 llvm::Type *ArgType = ArgValue->
getType();
2727 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2728 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
2729 Value *Inverse = Builder.CreateNot(ArgValue,
"not");
2730 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
2731 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
2732 Value *
Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
2733 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2737 case Builtin::BI__builtin_ctzs:
2738 case Builtin::BI__builtin_ctz:
2739 case Builtin::BI__builtin_ctzl:
2740 case Builtin::BI__builtin_ctzll: {
2743 llvm::Type *ArgType = ArgValue->
getType();
2747 Value *ZeroUndef = Builder.getInt1(
getTarget().isCLZForZeroUndef());
2748 Value *
Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2749 if (
Result->getType() != ResultType)
2750 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2754 case Builtin::BI__builtin_clzs:
2755 case Builtin::BI__builtin_clz:
2756 case Builtin::BI__builtin_clzl:
2757 case Builtin::BI__builtin_clzll: {
2760 llvm::Type *ArgType = ArgValue->
getType();
2764 Value *ZeroUndef = Builder.getInt1(
getTarget().isCLZForZeroUndef());
2765 Value *
Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2766 if (
Result->getType() != ResultType)
2767 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2771 case Builtin::BI__builtin_ffs:
2772 case Builtin::BI__builtin_ffsl:
2773 case Builtin::BI__builtin_ffsll: {
2777 llvm::Type *ArgType = ArgValue->
getType();
2782 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
2783 llvm::ConstantInt::get(ArgType, 1));
2784 Value *
Zero = llvm::Constant::getNullValue(ArgType);
2785 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
2786 Value *
Result = Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
2787 if (
Result->getType() != ResultType)
2788 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2792 case Builtin::BI__builtin_parity:
2793 case Builtin::BI__builtin_parityl:
2794 case Builtin::BI__builtin_parityll: {
2798 llvm::Type *ArgType = ArgValue->
getType();
2802 Value *Tmp = Builder.CreateCall(F, ArgValue);
2803 Value *
Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
2804 if (
Result->getType() != ResultType)
2805 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2809 case Builtin::BI__lzcnt16:
2810 case Builtin::BI__lzcnt:
2811 case Builtin::BI__lzcnt64: {
2814 llvm::Type *ArgType = ArgValue->
getType();
2818 Value *
Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
2819 if (
Result->getType() != ResultType)
2820 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2824 case Builtin::BI__popcnt16:
2825 case Builtin::BI__popcnt:
2826 case Builtin::BI__popcnt64:
2827 case Builtin::BI__builtin_popcount:
2828 case Builtin::BI__builtin_popcountl:
2829 case Builtin::BI__builtin_popcountll: {
2832 llvm::Type *ArgType = ArgValue->
getType();
2837 if (
Result->getType() != ResultType)
2838 Result = Builder.CreateIntCast(
Result, ResultType,
true,
2842 case Builtin::BI__builtin_unpredictable: {
2848 case Builtin::BI__builtin_expect: {
2850 llvm::Type *ArgType = ArgValue->
getType();
2861 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
2864 case Builtin::BI__builtin_expect_with_probability: {
2866 llvm::Type *ArgType = ArgValue->
getType();
2869 llvm::APFloat Probability(0.0);
2872 assert(EvalSucceed &&
"probability should be able to evaluate as float");
2874 bool LoseInfo =
false;
2875 Probability.convert(llvm::APFloat::IEEEdouble(),
2876 llvm::RoundingMode::Dynamic, &LoseInfo);
2878 Constant *Confidence = ConstantFP::get(Ty, Probability);
2885 Function *FnExpect =
2888 FnExpect, {ArgValue, ExpectedValue, Confidence},
"expval");
2891 case Builtin::BI__builtin_assume_aligned: {
2894 Value *OffsetValue =
2899 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
2900 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
2901 llvm::Value::MaximumAlignment);
2905 AlignmentCI, OffsetValue);
2908 case Builtin::BI__assume:
2909 case Builtin::BI__builtin_assume: {
2915 Builder.CreateCall(FnAssume, ArgValue);
2918 case Builtin::BI__builtin_assume_separate_storage: {
2925 Value *Values[] = {Value0, Value1};
2926 OperandBundleDefT<Value *> OBD(
"separate_storage", Values);
2927 Builder.CreateAssumption(ConstantInt::getTrue(
getLLVMContext()), {OBD});
2930 case Builtin::BI__arithmetic_fence: {
2933 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
2934 llvm::FastMathFlags FMF = Builder.getFastMathFlags();
2935 bool isArithmeticFenceEnabled =
2936 FMF.allowReassoc() &&
2939 if (ArgType->isComplexType()) {
2940 if (isArithmeticFenceEnabled) {
2943 Value *Real = Builder.CreateArithmeticFence(ComplexVal.first,
2945 Value *Imag = Builder.CreateArithmeticFence(ComplexVal.second,
2950 Value *Real = ComplexVal.first;
2951 Value *Imag = ComplexVal.second;
2955 if (isArithmeticFenceEnabled)
2957 Builder.CreateArithmeticFence(ArgValue,
ConvertType(ArgType)));
2960 case Builtin::BI__builtin_bswap16:
2961 case Builtin::BI__builtin_bswap32:
2962 case Builtin::BI__builtin_bswap64:
2963 case Builtin::BI_byteswap_ushort:
2964 case Builtin::BI_byteswap_ulong:
2965 case Builtin::BI_byteswap_uint64: {
2968 case Builtin::BI__builtin_bitreverse8:
2969 case Builtin::BI__builtin_bitreverse16:
2970 case Builtin::BI__builtin_bitreverse32:
2971 case Builtin::BI__builtin_bitreverse64: {
2974 case Builtin::BI__builtin_rotateleft8:
2975 case Builtin::BI__builtin_rotateleft16:
2976 case Builtin::BI__builtin_rotateleft32:
2977 case Builtin::BI__builtin_rotateleft64:
2978 case Builtin::BI_rotl8:
2979 case Builtin::BI_rotl16:
2980 case Builtin::BI_rotl:
2981 case Builtin::BI_lrotl:
2982 case Builtin::BI_rotl64:
2985 case Builtin::BI__builtin_rotateright8:
2986 case Builtin::BI__builtin_rotateright16:
2987 case Builtin::BI__builtin_rotateright32:
2988 case Builtin::BI__builtin_rotateright64:
2989 case Builtin::BI_rotr8:
2990 case Builtin::BI_rotr16:
2991 case Builtin::BI_rotr:
2992 case Builtin::BI_lrotr:
2993 case Builtin::BI_rotr64:
2996 case Builtin::BI__builtin_constant_p: {
3003 if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
3004 !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
3007 return RValue::get(ConstantInt::get(ResultType, 0));
3012 return RValue::get(ConstantInt::get(ResultType, 0));
3015 if (ArgType->isObjCObjectPointerType()) {
3019 ArgValue = Builder.CreateBitCast(ArgValue,
ConvertType(ArgType));
3024 if (
Result->getType() != ResultType)
3025 Result = Builder.CreateIntCast(
Result, ResultType,
false);
3028 case Builtin::BI__builtin_dynamic_object_size:
3029 case Builtin::BI__builtin_object_size: {
3036 bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
3038 nullptr, IsDynamic));
3040 case Builtin::BI__builtin_prefetch: {
3046 llvm::ConstantInt::get(
Int32Ty, 3);
3049 Builder.CreateCall(F, {Address, RW, Locality, Data});
3052 case Builtin::BI__builtin_readcyclecounter: {
3056 case Builtin::BI__builtin___clear_cache: {
3060 return RValue::get(Builder.CreateCall(F, {Begin, End}));
3062 case Builtin::BI__builtin_trap:
3065 case Builtin::BI__debugbreak:
3068 case Builtin::BI__builtin_unreachable: {
3077 case Builtin::BI__builtin_powi:
3078 case Builtin::BI__builtin_powif:
3079 case Builtin::BI__builtin_powil: {
3083 if (Builder.getIsFPConstrained()) {
3086 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3087 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_powi,
3089 return RValue::get(Builder.CreateConstrainedFPCall(F, { Src0, Src1 }));
3093 { Src0->getType(), Src1->getType() });
3094 return RValue::get(Builder.CreateCall(F, { Src0, Src1 }));
3096 case Builtin::BI__builtin_frexp:
3097 case Builtin::BI__builtin_frexpf:
3098 case Builtin::BI__builtin_frexpl:
3099 case Builtin::BI__builtin_frexpf128:
3100 case Builtin::BI__builtin_frexpf16:
3102 case Builtin::BI__builtin_isgreater:
3103 case Builtin::BI__builtin_isgreaterequal:
3104 case Builtin::BI__builtin_isless:
3105 case Builtin::BI__builtin_islessequal:
3106 case Builtin::BI__builtin_islessgreater:
3107 case Builtin::BI__builtin_isunordered: {
3110 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3114 switch (BuiltinID) {
3115 default: llvm_unreachable(
"Unknown ordered comparison");
3116 case Builtin::BI__builtin_isgreater:
3117 LHS = Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
3119 case Builtin::BI__builtin_isgreaterequal:
3120 LHS = Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
3122 case Builtin::BI__builtin_isless:
3123 LHS = Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
3125 case Builtin::BI__builtin_islessequal:
3126 LHS = Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
3128 case Builtin::BI__builtin_islessgreater:
3129 LHS = Builder.CreateFCmpONE(LHS, RHS,
"cmp");
3131 case Builtin::BI__builtin_isunordered:
3132 LHS = Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
3139 case Builtin::BI__builtin_isnan: {
3140 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3145 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcNan),
3149 case Builtin::BI__builtin_isinf: {
3150 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3155 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcInf),
3159 case Builtin::BIfinite:
3160 case Builtin::BI__finite:
3161 case Builtin::BIfinitef:
3162 case Builtin::BI__finitef:
3163 case Builtin::BIfinitel:
3164 case Builtin::BI__finitel:
3165 case Builtin::BI__builtin_isfinite: {
3166 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3171 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcFinite),
3175 case Builtin::BI__builtin_isnormal: {
3176 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3179 Builder.CreateZExt(Builder.createIsFPClass(
V, FPClassTest::fcNormal),
3183 case Builtin::BI__builtin_isfpclass: {
3188 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3190 return RValue::get(Builder.CreateZExt(Builder.createIsFPClass(
V, Test),
3194 case Builtin::BI__builtin_nondeterministic_value: {
3203 case Builtin::BI__builtin_elementwise_abs: {
3208 QT = VecTy->getElementType();
3210 Result = Builder.CreateBinaryIntrinsic(
3212 Builder.getFalse(),
nullptr,
"elt.abs");
3219 case Builtin::BI__builtin_elementwise_ceil:
3222 case Builtin::BI__builtin_elementwise_exp:
3225 case Builtin::BI__builtin_elementwise_exp2:
3228 case Builtin::BI__builtin_elementwise_log:
3231 case Builtin::BI__builtin_elementwise_log2:
3234 case Builtin::BI__builtin_elementwise_log10:
3237 case Builtin::BI__builtin_elementwise_pow: {
3240 case Builtin::BI__builtin_elementwise_cos:
3243 case Builtin::BI__builtin_elementwise_floor:
3246 case Builtin::BI__builtin_elementwise_roundeven:
3249 case Builtin::BI__builtin_elementwise_round:
3252 case Builtin::BI__builtin_elementwise_rint:
3255 case Builtin::BI__builtin_elementwise_nearbyint:
3258 case Builtin::BI__builtin_elementwise_sin:
3262 case Builtin::BI__builtin_elementwise_trunc:
3265 case Builtin::BI__builtin_elementwise_canonicalize:
3267 emitUnaryBuiltin(*
this, E, llvm::Intrinsic::canonicalize,
"elt.canonicalize"));
3268 case Builtin::BI__builtin_elementwise_copysign:
3270 case Builtin::BI__builtin_elementwise_fma:
3272 case Builtin::BI__builtin_elementwise_add_sat:
3273 case Builtin::BI__builtin_elementwise_sub_sat: {
3277 assert(Op0->
getType()->isIntOrIntVectorTy() &&
"integer type expected");
3280 Ty = VecTy->getElementType();
3283 if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
3284 Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
3286 Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;
3287 Result = Builder.CreateBinaryIntrinsic(Opc, Op0, Op1,
nullptr,
"elt.sat");
3291 case Builtin::BI__builtin_elementwise_max: {
3295 if (Op0->
getType()->isIntOrIntVectorTy()) {
3298 Ty = VecTy->getElementType();
3300 ? llvm::Intrinsic::smax
3301 : llvm::Intrinsic::umax,
3302 Op0, Op1,
nullptr,
"elt.max");
3304 Result = Builder.CreateMaxNum(Op0, Op1,
"elt.max");
3307 case Builtin::BI__builtin_elementwise_min: {
3311 if (Op0->
getType()->isIntOrIntVectorTy()) {
3314 Ty = VecTy->getElementType();
3316 ? llvm::Intrinsic::smin
3317 : llvm::Intrinsic::umin,
3318 Op0, Op1,
nullptr,
"elt.min");
3320 Result = Builder.CreateMinNum(Op0, Op1,
"elt.min");
3324 case Builtin::BI__builtin_reduce_max: {
3325 auto GetIntrinsicID = [](
QualType QT) {
3327 QT = VecTy->getElementType();
3329 return llvm::Intrinsic::vector_reduce_smax;
3331 return llvm::Intrinsic::vector_reduce_umax;
3333 return llvm::Intrinsic::vector_reduce_fmax;
3336 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3339 case Builtin::BI__builtin_reduce_min: {
3340 auto GetIntrinsicID = [](
QualType QT) {
3342 QT = VecTy->getElementType();
3344 return llvm::Intrinsic::vector_reduce_smin;
3346 return llvm::Intrinsic::vector_reduce_umin;
3348 return llvm::Intrinsic::vector_reduce_fmin;
3352 *
this, E, GetIntrinsicID(E->
getArg(0)->
getType()),
"rdx.min"));
3355 case Builtin::BI__builtin_reduce_add:
3357 *
this, E, llvm::Intrinsic::vector_reduce_add,
"rdx.add"));
3358 case Builtin::BI__builtin_reduce_mul:
3360 *
this, E, llvm::Intrinsic::vector_reduce_mul,
"rdx.mul"));
3361 case Builtin::BI__builtin_reduce_xor:
3363 *
this, E, llvm::Intrinsic::vector_reduce_xor,
"rdx.xor"));
3364 case Builtin::BI__builtin_reduce_or:
3366 *
this, E, llvm::Intrinsic::vector_reduce_or,
"rdx.or"));
3367 case Builtin::BI__builtin_reduce_and:
3369 *
this, E, llvm::Intrinsic::vector_reduce_and,
"rdx.and"));
3371 case Builtin::BI__builtin_matrix_transpose: {
3374 MatrixBuilder MB(Builder);
3375 Value *
Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
3376 MatrixTy->getNumColumns());
3380 case Builtin::BI__builtin_matrix_column_major_load: {
3381 MatrixBuilder MB(Builder);
3386 assert(PtrTy &&
"arg0 must be of pointer type");
3387 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3395 ResultTy->getNumRows(), ResultTy->getNumColumns(),
3400 case Builtin::BI__builtin_matrix_column_major_store: {
3401 MatrixBuilder MB(Builder);
3408 assert(PtrTy &&
"arg1 must be of pointer type");
3409 bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
3415 Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
3419 case Builtin::BI__builtin_isinf_sign: {
3421 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3425 Value *IsInf = Builder.CreateFCmpOEQ(
3426 AbsArg, ConstantFP::getInfinity(Arg->
getType()),
"isinf");
3432 Value *NegativeOne = ConstantInt::get(
IntTy, -1);
3433 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
3434 Value *
Result = Builder.CreateSelect(IsInf, SignResult, Zero);
3438 case Builtin::BI__builtin_flt_rounds: {
3443 if (
Result->getType() != ResultType)
3444 Result = Builder.CreateIntCast(
Result, ResultType,
true,
3449 case Builtin::BI__builtin_set_flt_rounds: {
3453 Builder.CreateCall(F,
V);
3457 case Builtin::BI__builtin_fpclassify: {
3458 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
3464 BasicBlock *
Begin = Builder.GetInsertBlock();
3466 Builder.SetInsertPoint(End);
3469 "fpclassify_result");
3472 Builder.SetInsertPoint(
Begin);
3473 Value *IsZero = Builder.CreateFCmpOEQ(
V, Constant::getNullValue(Ty),
3477 Builder.CreateCondBr(IsZero, End, NotZero);
3481 Builder.SetInsertPoint(NotZero);
3482 Value *IsNan = Builder.CreateFCmpUNO(
V,
V,
"cmp");
3485 Builder.CreateCondBr(IsNan, End, NotNan);
3486 Result->addIncoming(NanLiteral, NotZero);
3489 Builder.SetInsertPoint(NotNan);
3492 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(
V->getType()),
3496 Builder.CreateCondBr(IsInf, End, NotInf);
3497 Result->addIncoming(InfLiteral, NotNan);
3500 Builder.SetInsertPoint(NotInf);
3501 APFloat Smallest = APFloat::getSmallestNormalized(
3504 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(
V->getContext(), Smallest),
3506 Value *NormalResult =
3509 Builder.CreateBr(End);
3510 Result->addIncoming(NormalResult, NotInf);
3513 Builder.SetInsertPoint(End);
3517 case Builtin::BIalloca:
3518 case Builtin::BI_alloca:
3519 case Builtin::BI__builtin_alloca_uninitialized:
3520 case Builtin::BI__builtin_alloca: {
3524 const Align SuitableAlignmentInBytes =
3528 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3529 AI->setAlignment(SuitableAlignmentInBytes);
3530 if (BuiltinID != Builtin::BI__builtin_alloca_uninitialized)
3535 case Builtin::BI__builtin_alloca_with_align_uninitialized:
3536 case Builtin::BI__builtin_alloca_with_align: {
3540 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
3541 const Align AlignmentInBytes =
3543 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
3544 AI->setAlignment(AlignmentInBytes);
3545 if (BuiltinID != Builtin::BI__builtin_alloca_with_align_uninitialized)
3550 case Builtin::BIbzero:
3551 case Builtin::BI__builtin_bzero: {
3556 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal,
false);
3559 case Builtin::BImemcpy:
3560 case Builtin::BI__builtin_memcpy:
3561 case Builtin::BImempcpy:
3562 case Builtin::BI__builtin_mempcpy: {
3570 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
3571 if (BuiltinID == Builtin::BImempcpy ||
3572 BuiltinID == Builtin::BI__builtin_mempcpy)
3579 case Builtin::BI__builtin_memcpy_inline: {
3588 Builder.CreateMemCpyInline(Dest, Src, Size);
3592 case Builtin::BI__builtin_char_memchr:
3593 BuiltinID = Builtin::BI__builtin_memchr;
3596 case Builtin::BI__builtin___memcpy_chk: {
3603 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3604 if (
Size.ugt(DstSize))
3608 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3609 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
3613 case Builtin::BI__builtin_objc_memmove_collectable: {
3618 DestAddr, SrcAddr, SizeVal);
3622 case Builtin::BI__builtin___memmove_chk: {
3629 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3630 if (
Size.ugt(DstSize))
3634 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3635 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
3639 case Builtin::BImemmove:
3640 case Builtin::BI__builtin_memmove: {
3648 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
3651 case Builtin::BImemset:
3652 case Builtin::BI__builtin_memset: {
3655 Builder.getInt8Ty());
3659 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
3662 case Builtin::BI__builtin_memset_inline: {
3670 Builder.CreateMemSetInline(Dest, ByteVal, Size);
3673 case Builtin::BI__builtin___memset_chk: {
3680 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
3681 if (
Size.ugt(DstSize))
3685 Builder.getInt8Ty());
3686 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3687 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
3690 case Builtin::BI__builtin_wmemchr: {
3693 if (!
getTarget().getTriple().isOSMSVCRT())
3701 BasicBlock *Entry = Builder.GetInsertBlock();
3705 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(
SizeTy, 0));
3706 Builder.CreateCondBr(SizeEq0, Exit, CmpEq);
3709 PHINode *StrPhi = Builder.CreatePHI(Str->
getType(), 2);
3710 StrPhi->addIncoming(Str, Entry);
3711 PHINode *SizePhi = Builder.CreatePHI(
SizeTy, 2);
3712 SizePhi->addIncoming(Size, Entry);
3715 Value *StrCh = Builder.CreateAlignedLoad(WCharTy, StrPhi, WCharAlign);
3716 Value *FoundChr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 0);
3717 Value *StrEqChr = Builder.CreateICmpEQ(StrCh, Chr);
3718 Builder.CreateCondBr(StrEqChr, Exit,
Next);
3721 Value *NextStr = Builder.CreateConstInBoundsGEP1_32(WCharTy, StrPhi, 1);
3722 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(
SizeTy, 1));
3723 Value *NextSizeEq0 =
3724 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3725 Builder.CreateCondBr(NextSizeEq0, Exit, CmpEq);
3726 StrPhi->addIncoming(NextStr,
Next);
3727 SizePhi->addIncoming(NextSize,
Next);
3730 PHINode *
Ret = Builder.CreatePHI(Str->
getType(), 3);
3731 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()), Entry);
3732 Ret->addIncoming(llvm::Constant::getNullValue(Str->
getType()),
Next);
3733 Ret->addIncoming(FoundChr, CmpEq);
3736 case Builtin::BI__builtin_wmemcmp: {
3739 if (!
getTarget().getTriple().isOSMSVCRT())
3748 BasicBlock *Entry = Builder.GetInsertBlock();
3753 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(
SizeTy, 0));
3754 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
3757 PHINode *DstPhi = Builder.CreatePHI(Dst->
getType(), 2);
3758 DstPhi->addIncoming(Dst, Entry);
3759 PHINode *SrcPhi = Builder.CreatePHI(Src->
getType(), 2);
3760 SrcPhi->addIncoming(Src, Entry);
3761 PHINode *SizePhi = Builder.CreatePHI(
SizeTy, 2);
3762 SizePhi->addIncoming(Size, Entry);
3765 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
3766 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
3767 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
3768 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
3771 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
3772 Builder.CreateCondBr(DstLtSrc, Exit,
Next);
3775 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
3776 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
3777 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(
SizeTy, 1));
3778 Value *NextSizeEq0 =
3779 Builder.CreateICmpEQ(NextSize, ConstantInt::get(
SizeTy, 0));
3780 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
3781 DstPhi->addIncoming(NextDst,
Next);
3782 SrcPhi->addIncoming(NextSrc,
Next);
3783 SizePhi->addIncoming(NextSize,
Next);
3786 PHINode *
Ret = Builder.CreatePHI(
IntTy, 4);
3787 Ret->addIncoming(ConstantInt::get(
IntTy, 0), Entry);
3788 Ret->addIncoming(ConstantInt::get(
IntTy, 1), CmpGT);
3789 Ret->addIncoming(ConstantInt::get(
IntTy, -1), CmpLT);
3793 case Builtin::BI__builtin_dwarf_cfa: {
3808 case Builtin::BI__builtin_return_address: {
3814 case Builtin::BI_ReturnAddress: {
3816 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
3818 case Builtin::BI__builtin_frame_address: {
3824 case Builtin::BI__builtin_extract_return_addr: {
3829 case Builtin::BI__builtin_frob_return_addr: {
3834 case Builtin::BI__builtin_dwarf_sp_column: {
3835 llvm::IntegerType *Ty
3842 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
3844 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
3850 case Builtin::BI__builtin_eh_return: {
3855 assert((
IntTy->getBitWidth() == 32 ||
IntTy->getBitWidth() == 64) &&
3856 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
3859 : Intrinsic::eh_return_i64);
3860 Builder.CreateCall(F, {Int, Ptr});
3861 Builder.CreateUnreachable();
3868 case Builtin::BI__builtin_unwind_init: {
3870 Builder.CreateCall(F);
3873 case Builtin::BI__builtin_extend_pointer: {
3898 case Builtin::BI__builtin_setjmp: {
3903 Value *FrameAddr = Builder.CreateCall(
3905 ConstantInt::get(
Int32Ty, 0));
3906 Builder.CreateStore(FrameAddr, Buf);
3911 Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
3912 Builder.CreateStore(StackAddr, StackSaveSlot);
3918 case Builtin::BI__builtin_longjmp: {
3920 Buf = Builder.CreateBitCast(Buf,
Int8PtrTy);
3923 Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
3926 Builder.CreateUnreachable();
3933 case Builtin::BI__builtin_launder: {
3938 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
3942 case Builtin::BI__sync_fetch_and_add:
3943 case Builtin::BI__sync_fetch_and_sub:
3944 case Builtin::BI__sync_fetch_and_or:
3945 case Builtin::BI__sync_fetch_and_and:
3946 case Builtin::BI__sync_fetch_and_xor:
3947 case Builtin::BI__sync_fetch_and_nand:
3948 case Builtin::BI__sync_add_and_fetch:
3949 case Builtin::BI__sync_sub_and_fetch:
3950 case Builtin::BI__sync_and_and_fetch:
3951 case Builtin::BI__sync_or_and_fetch:
3952 case Builtin::BI__sync_xor_and_fetch:
3953 case Builtin::BI__sync_nand_and_fetch:
3954 case Builtin::BI__sync_val_compare_and_swap:
3955 case Builtin::BI__sync_bool_compare_and_swap:
3956 case Builtin::BI__sync_lock_test_and_set:
3957 case Builtin::BI__sync_lock_release:
3958 case Builtin::BI__sync_swap:
3959 llvm_unreachable(
"Shouldn't make it through sema");
3960 case Builtin::BI__sync_fetch_and_add_1:
3961 case Builtin::BI__sync_fetch_and_add_2:
3962 case Builtin::BI__sync_fetch_and_add_4:
3963 case Builtin::BI__sync_fetch_and_add_8:
3964 case Builtin::BI__sync_fetch_and_add_16:
3966 case Builtin::BI__sync_fetch_and_sub_1:
3967 case Builtin::BI__sync_fetch_and_sub_2:
3968 case Builtin::BI__sync_fetch_and_sub_4:
3969 case Builtin::BI__sync_fetch_and_sub_8:
3970 case Builtin::BI__sync_fetch_and_sub_16:
3972 case Builtin::BI__sync_fetch_and_or_1:
3973 case Builtin::BI__sync_fetch_and_or_2:
3974 case Builtin::BI__sync_fetch_and_or_4:
3975 case Builtin::BI__sync_fetch_and_or_8:
3976 case Builtin::BI__sync_fetch_and_or_16:
3978 case Builtin::BI__sync_fetch_and_and_1:
3979 case Builtin::BI__sync_fetch_and_and_2:
3980 case Builtin::BI__sync_fetch_and_and_4:
3981 case Builtin::BI__sync_fetch_and_and_8:
3982 case Builtin::BI__sync_fetch_and_and_16:
3984 case Builtin::BI__sync_fetch_and_xor_1:
3985 case Builtin::BI__sync_fetch_and_xor_2:
3986 case Builtin::BI__sync_fetch_and_xor_4:
3987 case Builtin::BI__sync_fetch_and_xor_8:
3988 case Builtin::BI__sync_fetch_and_xor_16:
3990 case Builtin::BI__sync_fetch_and_nand_1:
3991 case Builtin::BI__sync_fetch_and_nand_2:
3992 case Builtin::BI__sync_fetch_and_nand_4:
3993 case Builtin::BI__sync_fetch_and_nand_8:
3994 case Builtin::BI__sync_fetch_and_nand_16:
3998 case Builtin::BI__sync_fetch_and_min:
4000 case Builtin::BI__sync_fetch_and_max:
4002 case Builtin::BI__sync_fetch_and_umin:
4004 case Builtin::BI__sync_fetch_and_umax:
4007 case Builtin::BI__sync_add_and_fetch_1:
4008 case Builtin::BI__sync_add_and_fetch_2:
4009 case Builtin::BI__sync_add_and_fetch_4:
4010 case Builtin::BI__sync_add_and_fetch_8:
4011 case Builtin::BI__sync_add_and_fetch_16:
4013 llvm::Instruction::Add);
4014 case Builtin::BI__sync_sub_and_fetch_1:
4015 case Builtin::BI__sync_sub_and_fetch_2:
4016 case Builtin::BI__sync_sub_and_fetch_4:
4017 case Builtin::BI__sync_sub_and_fetch_8:
4018 case Builtin::BI__sync_sub_and_fetch_16:
4020 llvm::Instruction::Sub);
4021 case Builtin::BI__sync_and_and_fetch_1:
4022 case Builtin::BI__sync_and_and_fetch_2:
4023 case Builtin::BI__sync_and_and_fetch_4:
4024 case Builtin::BI__sync_and_and_fetch_8:
4025 case Builtin::BI__sync_and_and_fetch_16:
4027 llvm::Instruction::And);
4028 case Builtin::BI__sync_or_and_fetch_1:
4029 case Builtin::BI__sync_or_and_fetch_2:
4030 case Builtin::BI__sync_or_and_fetch_4:
4031 case Builtin::BI__sync_or_and_fetch_8:
4032 case Builtin::BI__sync_or_and_fetch_16:
4034 llvm::Instruction::Or);
4035 case Builtin::BI__sync_xor_and_fetch_1:
4036 case Builtin::BI__sync_xor_and_fetch_2:
4037 case Builtin::BI__sync_xor_and_fetch_4:
4038 case Builtin::BI__sync_xor_and_fetch_8:
4039 case Builtin::BI__sync_xor_and_fetch_16:
4041 llvm::Instruction::Xor);
4042 case Builtin::BI__sync_nand_and_fetch_1:
4043 case Builtin::BI__sync_nand_and_fetch_2:
4044 case Builtin::BI__sync_nand_and_fetch_4:
4045 case Builtin::BI__sync_nand_and_fetch_8:
4046 case Builtin::BI__sync_nand_and_fetch_16:
4048 llvm::Instruction::And,
true);
4050 case Builtin::BI__sync_val_compare_and_swap_1:
4051 case Builtin::BI__sync_val_compare_and_swap_2:
4052 case Builtin::BI__sync_val_compare_and_swap_4:
4053 case Builtin::BI__sync_val_compare_and_swap_8:
4054 case Builtin::BI__sync_val_compare_and_swap_16:
4057 case Builtin::BI__sync_bool_compare_and_swap_1:
4058 case Builtin::BI__sync_bool_compare_and_swap_2:
4059 case Builtin::BI__sync_bool_compare_and_swap_4:
4060 case Builtin::BI__sync_bool_compare_and_swap_8:
4061 case Builtin::BI__sync_bool_compare_and_swap_16:
4064 case Builtin::BI__sync_swap_1:
4065 case Builtin::BI__sync_swap_2:
4066 case Builtin::BI__sync_swap_4:
4067 case Builtin::BI__sync_swap_8:
4068 case Builtin::BI__sync_swap_16:
4071 case Builtin::BI__sync_lock_test_and_set_1:
4072 case Builtin::BI__sync_lock_test_and_set_2:
4073 case Builtin::BI__sync_lock_test_and_set_4:
4074 case Builtin::BI__sync_lock_test_and_set_8:
4075 case Builtin::BI__sync_lock_test_and_set_16:
4078 case Builtin::BI__sync_lock_release_1:
4079 case Builtin::BI__sync_lock_release_2:
4080 case Builtin::BI__sync_lock_release_4:
4081 case Builtin::BI__sync_lock_release_8:
4082 case Builtin::BI__sync_lock_release_16: {
4088 llvm::StoreInst *
Store =
4089 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
4091 Store->setAtomic(llvm::AtomicOrdering::Release);
4095 case Builtin::BI__sync_synchronize: {
4103 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
4107 case Builtin::BI__builtin_nontemporal_load:
4109 case Builtin::BI__builtin_nontemporal_store:
4111 case Builtin::BI__c11_atomic_is_lock_free:
4112 case Builtin::BI__atomic_is_lock_free: {
4116 const char *LibCallName =
"__atomic_is_lock_free";
4120 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
4134 case Builtin::BI__atomic_test_and_set: {
4142 Value *NewVal = Builder.getInt8(1);
4144 if (isa<llvm::ConstantInt>(Order)) {
4146 AtomicRMWInst *
Result =
nullptr;
4150 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4151 llvm::AtomicOrdering::Monotonic);
4155 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4156 llvm::AtomicOrdering::Acquire);
4159 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4160 llvm::AtomicOrdering::Release);
4164 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4165 llvm::AtomicOrdering::AcquireRelease);
4168 Result = Builder.CreateAtomicRMW(
4169 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
4170 llvm::AtomicOrdering::SequentiallyConsistent);
4173 Result->setVolatile(Volatile);
4179 llvm::BasicBlock *BBs[5] = {
4186 llvm::AtomicOrdering Orders[5] = {
4187 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
4188 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
4189 llvm::AtomicOrdering::SequentiallyConsistent};
4191 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4192 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4194 Builder.SetInsertPoint(ContBB);
4195 PHINode *
Result = Builder.CreatePHI(
Int8Ty, 5,
"was_set");
4197 for (
unsigned i = 0; i < 5; ++i) {
4198 Builder.SetInsertPoint(BBs[i]);
4199 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
4200 Ptr, NewVal, Orders[i]);
4201 RMW->setVolatile(Volatile);
4202 Result->addIncoming(RMW, BBs[i]);
4203 Builder.CreateBr(ContBB);
4206 SI->addCase(Builder.getInt32(0), BBs[0]);
4207 SI->addCase(Builder.getInt32(1), BBs[1]);
4208 SI->addCase(Builder.getInt32(2), BBs[1]);
4209 SI->addCase(Builder.getInt32(3), BBs[2]);
4210 SI->addCase(Builder.getInt32(4), BBs[3]);
4211 SI->addCase(Builder.getInt32(5), BBs[4]);
4213 Builder.SetInsertPoint(ContBB);
4217 case Builtin::BI__atomic_clear: {
4220 PtrTy->castAs<
PointerType>()->getPointeeType().isVolatileQualified();
4224 Value *NewVal = Builder.getInt8(0);
4226 if (isa<llvm::ConstantInt>(Order)) {
4228 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4232 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
4235 Store->setOrdering(llvm::AtomicOrdering::Release);
4238 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
4246 llvm::BasicBlock *BBs[3] = {
4251 llvm::AtomicOrdering Orders[3] = {
4252 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
4253 llvm::AtomicOrdering::SequentiallyConsistent};
4255 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4256 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
4258 for (
unsigned i = 0; i < 3; ++i) {
4259 Builder.SetInsertPoint(BBs[i]);
4260 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
4261 Store->setOrdering(Orders[i]);
4262 Builder.CreateBr(ContBB);
4265 SI->addCase(Builder.getInt32(0), BBs[0]);
4266 SI->addCase(Builder.getInt32(3), BBs[1]);
4267 SI->addCase(Builder.getInt32(5), BBs[2]);
4269 Builder.SetInsertPoint(ContBB);
4273 case Builtin::BI__atomic_thread_fence:
4274 case Builtin::BI__atomic_signal_fence:
4275 case Builtin::BI__c11_atomic_thread_fence:
4276 case Builtin::BI__c11_atomic_signal_fence: {
4277 llvm::SyncScope::ID SSID;
4278 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
4279 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
4280 SSID = llvm::SyncScope::SingleThread;
4282 SSID = llvm::SyncScope::System;
4284 if (isa<llvm::ConstantInt>(Order)) {
4292 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4295 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4298 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4301 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4307 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
4314 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
4315 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
4317 Builder.SetInsertPoint(AcquireBB);
4318 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
4319 Builder.CreateBr(ContBB);
4320 SI->addCase(Builder.getInt32(1), AcquireBB);
4321 SI->addCase(Builder.getInt32(2), AcquireBB);
4323 Builder.SetInsertPoint(ReleaseBB);
4324 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
4325 Builder.CreateBr(ContBB);
4326 SI->addCase(Builder.getInt32(3), ReleaseBB);
4328 Builder.SetInsertPoint(AcqRelBB);
4329 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
4330 Builder.CreateBr(ContBB);
4331 SI->addCase(Builder.getInt32(4), AcqRelBB);
4333 Builder.SetInsertPoint(SeqCstBB);
4334 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
4335 Builder.CreateBr(ContBB);
4336 SI->addCase(Builder.getInt32(5), SeqCstBB);
4338 Builder.SetInsertPoint(ContBB);
4342 case Builtin::BI__builtin_signbit:
4343 case Builtin::BI__builtin_signbitf:
4344 case Builtin::BI__builtin_signbitl: {
4349 case Builtin::BI__warn_memset_zero_len:
4351 case Builtin::BI__annotation: {
4356 assert(Str->getCharByteWidth() == 2);
4357 StringRef WideBytes = Str->getBytes();
4358 std::string StrUtf8;
4359 if (!convertUTF16ToUTF8String(
4360 ArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
4364 Strings.push_back(llvm::MDString::get(
getLLVMContext(), StrUtf8));
4371 Builder.CreateCall(F, MetadataAsValue::get(
getLLVMContext(), StrTuple));
4374 case Builtin::BI__builtin_annotation: {
4387 case Builtin::BI__builtin_addcb:
4388 case Builtin::BI__builtin_addcs:
4389 case Builtin::BI__builtin_addc:
4390 case Builtin::BI__builtin_addcl:
4391 case Builtin::BI__builtin_addcll:
4392 case Builtin::BI__builtin_subcb:
4393 case Builtin::BI__builtin_subcs:
4394 case Builtin::BI__builtin_subc:
4395 case Builtin::BI__builtin_subcl:
4396 case Builtin::BI__builtin_subcll: {
4422 llvm::Intrinsic::ID IntrinsicId;
4423 switch (BuiltinID) {
4424 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
4425 case Builtin::BI__builtin_addcb:
4426 case Builtin::BI__builtin_addcs:
4427 case Builtin::BI__builtin_addc:
4428 case Builtin::BI__builtin_addcl:
4429 case Builtin::BI__builtin_addcll:
4430 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4432 case Builtin::BI__builtin_subcb:
4433 case Builtin::BI__builtin_subcs:
4434 case Builtin::BI__builtin_subc:
4435 case Builtin::BI__builtin_subcl:
4436 case Builtin::BI__builtin_subcll:
4437 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4442 llvm::Value *Carry1;
4445 llvm::Value *Carry2;
4447 Sum1, Carryin, Carry2);
4448 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
4450 Builder.CreateStore(CarryOut, CarryOutPtr);
4454 case Builtin::BI__builtin_add_overflow:
4455 case Builtin::BI__builtin_sub_overflow:
4456 case Builtin::BI__builtin_mul_overflow: {
4464 WidthAndSignedness LeftInfo =
4466 WidthAndSignedness RightInfo =
4468 WidthAndSignedness ResultInfo =
4475 RightInfo, ResultArg, ResultQTy,
4481 *
this, LeftArg, LeftInfo, RightArg, RightInfo, ResultArg, ResultQTy,
4484 WidthAndSignedness EncompassingInfo =
4487 llvm::Type *EncompassingLLVMTy =
4492 llvm::Intrinsic::ID IntrinsicId;
4493 switch (BuiltinID) {
4495 llvm_unreachable(
"Unknown overflow builtin id.");
4496 case Builtin::BI__builtin_add_overflow:
4497 IntrinsicId = EncompassingInfo.Signed
4498 ? llvm::Intrinsic::sadd_with_overflow
4499 : llvm::Intrinsic::uadd_with_overflow;
4501 case Builtin::BI__builtin_sub_overflow:
4502 IntrinsicId = EncompassingInfo.Signed
4503 ? llvm::Intrinsic::ssub_with_overflow
4504 : llvm::Intrinsic::usub_with_overflow;
4506 case Builtin::BI__builtin_mul_overflow:
4507 IntrinsicId = EncompassingInfo.Signed
4508 ? llvm::Intrinsic::smul_with_overflow
4509 : llvm::Intrinsic::umul_with_overflow;
4518 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
4519 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
4522 llvm::Value *Overflow, *
Result;
4525 if (EncompassingInfo.Width > ResultInfo.Width) {
4528 llvm::Value *ResultTrunc = Builder.CreateTrunc(
Result, ResultLLVMTy);
4532 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
4533 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
4534 llvm::Value *TruncationOverflow =
4535 Builder.CreateICmpNE(
Result, ResultTruncExt);
4537 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
4549 case Builtin::BI__builtin_uadd_overflow:
4550 case Builtin::BI__builtin_uaddl_overflow:
4551 case Builtin::BI__builtin_uaddll_overflow:
4552 case Builtin::BI__builtin_usub_overflow:
4553 case Builtin::BI__builtin_usubl_overflow:
4554 case Builtin::BI__builtin_usubll_overflow:
4555 case Builtin::BI__builtin_umul_overflow:
4556 case Builtin::BI__builtin_umull_overflow:
4557 case Builtin::BI__builtin_umulll_overflow:
4558 case Builtin::BI__builtin_sadd_overflow:
4559 case Builtin::BI__builtin_saddl_overflow:
4560 case Builtin::BI__builtin_saddll_overflow:
4561 case Builtin::BI__builtin_ssub_overflow:
4562 case Builtin::BI__builtin_ssubl_overflow:
4563 case Builtin::BI__builtin_ssubll_overflow:
4564 case Builtin::BI__builtin_smul_overflow:
4565 case Builtin::BI__builtin_smull_overflow:
4566 case Builtin::BI__builtin_smulll_overflow: {
4576 llvm::Intrinsic::ID IntrinsicId;
4577 switch (BuiltinID) {
4578 default: llvm_unreachable(
"Unknown overflow builtin id.");
4579 case Builtin::BI__builtin_uadd_overflow:
4580 case Builtin::BI__builtin_uaddl_overflow:
4581 case Builtin::BI__builtin_uaddll_overflow:
4582 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
4584 case Builtin::BI__builtin_usub_overflow:
4585 case Builtin::BI__builtin_usubl_overflow:
4586 case Builtin::BI__builtin_usubll_overflow:
4587 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
4589 case Builtin::BI__builtin_umul_overflow:
4590 case Builtin::BI__builtin_umull_overflow:
4591 case Builtin::BI__builtin_umulll_overflow:
4592 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
4594 case Builtin::BI__builtin_sadd_overflow:
4595 case Builtin::BI__builtin_saddl_overflow:
4596 case Builtin::BI__builtin_saddll_overflow:
4597 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
4599 case Builtin::BI__builtin_ssub_overflow:
4600 case Builtin::BI__builtin_ssubl_overflow:
4601 case Builtin::BI__builtin_ssubll_overflow:
4602 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
4604 case Builtin::BI__builtin_smul_overflow:
4605 case Builtin::BI__builtin_smull_overflow:
4606 case Builtin::BI__builtin_smulll_overflow:
4607 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
4614 Builder.CreateStore(Sum, SumOutPtr);
4618 case Builtin::BIaddressof:
4619 case Builtin::BI__addressof:
4620 case Builtin::BI__builtin_addressof:
4622 case Builtin::BI__builtin_function_start:
4625 case Builtin::BI__builtin_operator_new:
4628 case Builtin::BI__builtin_operator_delete:
4633 case Builtin::BI__builtin_is_aligned:
4635 case Builtin::BI__builtin_align_up:
4637 case Builtin::BI__builtin_align_down:
4640 case Builtin::BI__noop:
4643 case Builtin::BI__builtin_call_with_static_chain: {
4650 case Builtin::BI_InterlockedExchange8:
4651 case Builtin::BI_InterlockedExchange16:
4652 case Builtin::BI_InterlockedExchange:
4653 case Builtin::BI_InterlockedExchangePointer:
4656 case Builtin::BI_InterlockedCompareExchangePointer:
4657 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
4659 llvm::IntegerType *IntType = IntegerType::get(
4665 RTy = Exchange->getType();
4666 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
4668 llvm::Value *Comparand =
4672 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
4673 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
4675 auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
4676 Ordering, Ordering);
4677 Result->setVolatile(
true);
4683 case Builtin::BI_InterlockedCompareExchange8:
4684 case Builtin::BI_InterlockedCompareExchange16:
4685 case Builtin::BI_InterlockedCompareExchange:
4686 case Builtin::BI_InterlockedCompareExchange64:
4688 case Builtin::BI_InterlockedIncrement16:
4689 case Builtin::BI_InterlockedIncrement:
4692 case Builtin::BI_InterlockedDecrement16:
4693 case Builtin::BI_InterlockedDecrement:
4696 case Builtin::BI_InterlockedAnd8:
4697 case Builtin::BI_InterlockedAnd16:
4698 case Builtin::BI_InterlockedAnd:
4700 case Builtin::BI_InterlockedExchangeAdd8:
4701 case Builtin::BI_InterlockedExchangeAdd16:
4702 case Builtin::BI_InterlockedExchangeAdd:
4705 case Builtin::BI_InterlockedExchangeSub8:
4706 case Builtin::BI_InterlockedExchangeSub16:
4707 case Builtin::BI_InterlockedExchangeSub:
4710 case Builtin::BI_InterlockedOr8:
4711 case Builtin::BI_InterlockedOr16:
4712 case Builtin::BI_InterlockedOr:
4714 case Builtin::BI_InterlockedXor8:
4715 case Builtin::BI_InterlockedXor16:
4716 case Builtin::BI_InterlockedXor:
4719 case Builtin::BI_bittest64:
4720 case Builtin::BI_bittest:
4721 case Builtin::BI_bittestandcomplement64:
4722 case Builtin::BI_bittestandcomplement:
4723 case Builtin::BI_bittestandreset64:
4724 case Builtin::BI_bittestandreset:
4725 case Builtin::BI_bittestandset64:
4726 case Builtin::BI_bittestandset:
4727 case Builtin::BI_interlockedbittestandreset:
4728 case Builtin::BI_interlockedbittestandreset64:
4729 case Builtin::BI_interlockedbittestandset64:
4730 case Builtin::BI_interlockedbittestandset:
4731 case Builtin::BI_interlockedbittestandset_acq:
4732 case Builtin::BI_interlockedbittestandset_rel:
4733 case Builtin::BI_interlockedbittestandset_nf:
4734 case Builtin::BI_interlockedbittestandreset_acq:
4735 case Builtin::BI_interlockedbittestandreset_rel:
4736 case Builtin::BI_interlockedbittestandreset_nf:
4741 case Builtin::BI__iso_volatile_load8:
4742 case Builtin::BI__iso_volatile_load16:
4743 case Builtin::BI__iso_volatile_load32:
4744 case Builtin::BI__iso_volatile_load64:
4746 case Builtin::BI__iso_volatile_store8:
4747 case Builtin::BI__iso_volatile_store16:
4748 case Builtin::BI__iso_volatile_store32:
4749 case Builtin::BI__iso_volatile_store64:
4752 case Builtin::BI__exception_code:
4753 case Builtin::BI_exception_code:
4755 case Builtin::BI__exception_info:
4756 case Builtin::BI_exception_info:
4758 case Builtin::BI__abnormal_termination:
4759 case Builtin::BI_abnormal_termination:
4761 case Builtin::BI_setjmpex:
4766 case Builtin::BI_setjmp:
4769 if (
getTarget().getTriple().getArch() == llvm::Triple::x86)
4771 else if (
getTarget().getTriple().getArch() == llvm::Triple::aarch64)
4778 case Builtin::BImove:
4779 case Builtin::BImove_if_noexcept:
4780 case Builtin::BIforward:
4781 case Builtin::BIforward_like:
4782 case Builtin::BIas_const:
4784 case Builtin::BI__GetExceptionInfo: {
4785 if (llvm::GlobalVariable *GV =
4791 case Builtin::BI__fastfail:
4794 case Builtin::BI__builtin_coro_id:
4796 case Builtin::BI__builtin_coro_promise:
4798 case Builtin::BI__builtin_coro_resume:
4801 case Builtin::BI__builtin_coro_frame:
4803 case Builtin::BI__builtin_coro_noop:
4805 case Builtin::BI__builtin_coro_free:
4807 case Builtin::BI__builtin_coro_destroy:
4810 case Builtin::BI__builtin_coro_done:
4812 case Builtin::BI__builtin_coro_alloc:
4814 case Builtin::BI__builtin_coro_begin:
4816 case Builtin::BI__builtin_coro_end:
4818 case Builtin::BI__builtin_coro_suspend:
4820 case Builtin::BI__builtin_coro_size:
4822 case Builtin::BI__builtin_coro_align:
4826 case Builtin::BIread_pipe:
4827 case Builtin::BIwrite_pipe: {
4831 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4832 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4835 unsigned GenericAS =
4837 llvm::Type *I8PTy = llvm::PointerType::get(
4842 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2"
4847 llvm::FunctionType *FTy = llvm::FunctionType::get(
4849 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
4852 {Arg0, BCast, PacketSize, PacketAlign}));
4855 "Illegal number of parameters to pipe function");
4856 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4"
4863 llvm::FunctionType *FTy = llvm::FunctionType::get(
4865 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
4869 Arg2 = Builder.CreateZExtOrTrunc(Arg2,
Int32Ty);
4872 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
4877 case Builtin::BIreserve_read_pipe:
4878 case Builtin::BIreserve_write_pipe:
4879 case Builtin::BIwork_group_reserve_read_pipe:
4880 case Builtin::BIwork_group_reserve_write_pipe:
4881 case Builtin::BIsub_group_reserve_read_pipe:
4882 case Builtin::BIsub_group_reserve_write_pipe: {
4885 if (BuiltinID == Builtin::BIreserve_read_pipe)
4886 Name =
"__reserve_read_pipe";
4887 else if (BuiltinID == Builtin::BIreserve_write_pipe)
4888 Name =
"__reserve_write_pipe";
4889 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
4890 Name =
"__work_group_reserve_read_pipe";
4891 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
4892 Name =
"__work_group_reserve_write_pipe";
4893 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
4894 Name =
"__sub_group_reserve_read_pipe";
4896 Name =
"__sub_group_reserve_write_pipe";
4902 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4903 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4907 llvm::FunctionType *FTy = llvm::FunctionType::get(
4912 Arg1 = Builder.CreateZExtOrTrunc(Arg1,
Int32Ty);
4914 {Arg0, Arg1, PacketSize, PacketAlign}));
4918 case Builtin::BIcommit_read_pipe:
4919 case Builtin::BIcommit_write_pipe:
4920 case Builtin::BIwork_group_commit_read_pipe:
4921 case Builtin::BIwork_group_commit_write_pipe:
4922 case Builtin::BIsub_group_commit_read_pipe:
4923 case Builtin::BIsub_group_commit_write_pipe: {
4925 if (BuiltinID == Builtin::BIcommit_read_pipe)
4926 Name =
"__commit_read_pipe";
4927 else if (BuiltinID == Builtin::BIcommit_write_pipe)
4928 Name =
"__commit_write_pipe";
4929 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
4930 Name =
"__work_group_commit_read_pipe";
4931 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
4932 Name =
"__work_group_commit_write_pipe";
4933 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
4934 Name =
"__sub_group_commit_read_pipe";
4936 Name =
"__sub_group_commit_write_pipe";
4941 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4942 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4946 llvm::FunctionType *FTy =
4951 {Arg0, Arg1, PacketSize, PacketAlign}));
4954 case Builtin::BIget_pipe_num_packets:
4955 case Builtin::BIget_pipe_max_packets: {
4956 const char *BaseName;
4958 if (BuiltinID == Builtin::BIget_pipe_num_packets)
4959 BaseName =
"__get_pipe_num_packets";
4961 BaseName =
"__get_pipe_max_packets";
4962 std::string Name = std::string(BaseName) +
4963 std::string(PipeTy->isReadOnly() ?
"_ro" :
"_wo");
4968 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
4969 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
4971 llvm::FunctionType *FTy = llvm::FunctionType::get(
4975 {Arg0, PacketSize, PacketAlign}));
4979 case Builtin::BIto_global:
4980 case Builtin::BIto_local:
4981 case Builtin::BIto_private: {
4983 auto NewArgT = llvm::PointerType::get(
Int8Ty,
4985 auto NewRetT = llvm::PointerType::get(
Int8Ty,
4988 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
4989 llvm::Value *NewArg;
4990 if (Arg0->
getType()->getPointerAddressSpace() !=
4991 NewArgT->getPointerAddressSpace())
4992 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
4994 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
4998 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
5004 case Builtin::BIenqueue_kernel: {
5009 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5015 llvm::Value *
Range = NDRangeL.getAddress(*this).getPointer();
5016 llvm::Type *RangeTy = NDRangeL.getAddress(*this).getType();
5021 Name =
"__enqueue_kernel_basic";
5022 llvm::Type *ArgTys[] = {QueueTy,
Int32Ty, RangeTy, GenericVoidPtrTy,
5024 llvm::FunctionType *FTy = llvm::FunctionType::get(
5030 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5031 llvm::Value *
Block =
5032 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5034 AttrBuilder B(Builder.getContext());
5035 B.addByValAttr(NDRangeL.getAddress(*this).getElementType());
5036 llvm::AttributeList ByValAttrSet =
5037 llvm::AttributeList::get(
CGM.
getModule().getContext(), 3U, B);
5041 {Queue, Flags, Range, Kernel, Block});
5042 RTCall->setAttributes(ByValAttrSet);
5045 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
5049 auto CreateArrayForSizeVar = [=](
unsigned First)
5050 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
5051 llvm::APInt ArraySize(32, NumArgs -
First);
5056 llvm::Value *TmpPtr = Tmp.getPointer();
5059 llvm::Value *ElemPtr;
5062 auto *
Zero = llvm::ConstantInt::get(
IntTy, 0);
5063 for (
unsigned I =
First; I < NumArgs; ++I) {
5064 auto *Index = llvm::ConstantInt::get(
IntTy, I -
First);
5065 auto *GEP = Builder.CreateGEP(Tmp.getElementType(), TmpPtr,
5071 Builder.CreateAlignedStore(
5074 return std::tie(ElemPtr, TmpSize, TmpPtr);
5080 Name =
"__enqueue_kernel_varargs";
5084 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5085 auto *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5086 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5087 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
5091 llvm::Value *
const Args[] = {Queue, Flags,
5095 llvm::Type *
const ArgTys[] = {
5096 QueueTy,
IntTy, RangeTy, GenericVoidPtrTy,
5097 GenericVoidPtrTy,
IntTy, ElemPtr->getType()};
5099 llvm::FunctionType *FTy = llvm::FunctionType::get(
Int32Ty, ArgTys,
false);
5108 llvm::PointerType *PtrTy = llvm::PointerType::get(
5112 llvm::Value *NumEvents =
5118 llvm::Value *EventWaitList =
nullptr;
5121 EventWaitList = llvm::ConstantPointerNull::get(PtrTy);
5127 EventWaitList = Builder.CreatePointerCast(EventWaitList, PtrTy);
5129 llvm::Value *EventRet =
nullptr;
5132 EventRet = llvm::ConstantPointerNull::get(PtrTy);
5141 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5142 llvm::Value *
Block =
5143 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5145 std::vector<llvm::Type *> ArgTys = {
5147 PtrTy, PtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
5149 std::vector<llvm::Value *> Args = {Queue, Flags,
Range,
5150 NumEvents, EventWaitList, EventRet,
5155 Name =
"__enqueue_kernel_basic_events";
5156 llvm::FunctionType *FTy = llvm::FunctionType::get(
5164 Args.push_back(ConstantInt::get(
Int32Ty, NumArgs - 7));
5166 Name =
"__enqueue_kernel_events_varargs";
5168 llvm::Value *ElemPtr, *TmpSize, *TmpPtr;
5169 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
5170 Args.push_back(ElemPtr);
5171 ArgTys.push_back(ElemPtr->getType());
5173 llvm::FunctionType *FTy = llvm::FunctionType::get(
5186 case Builtin::BIget_kernel_work_group_size: {
5187 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5192 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5193 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5196 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5198 "__get_kernel_work_group_size_impl"),
5201 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
5202 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5207 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5208 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5211 llvm::FunctionType::get(
IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
5213 "__get_kernel_preferred_work_group_size_multiple_impl"),
5216 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
5217 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
5218 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
5221 llvm::Value *NDRange = NDRangeL.getAddress(*this).getPointer();
5225 Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
5226 Value *
Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
5228 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
5229 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl"
5230 :
"__get_kernel_sub_group_count_for_ndrange_impl";
5233 llvm::FunctionType::get(
5234 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
5237 {NDRange, Kernel, Block}));
5240 case Builtin::BI__builtin_store_half:
5241 case Builtin::BI__builtin_store_halff: {
5244 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
5245 Builder.CreateStore(HalfVal,
Address);
5248 case Builtin::BI__builtin_load_half: {
5251 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
5253 case Builtin::BI__builtin_load_halff: {
5256 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
5258 case Builtin::BIprintf:
5259 if (
getTarget().getTriple().isNVPTX() ||
5270 case Builtin::BI__builtin_canonicalize:
5271 case Builtin::BI__builtin_canonicalizef:
5272 case Builtin::BI__builtin_canonicalizef16:
5273 case Builtin::BI__builtin_canonicalizel:
5276 case Builtin::BI__builtin_thread_pointer: {
5277 if (!
getContext().getTargetInfo().isTLSSupported())
5282 case Builtin::BI__builtin_os_log_format:
5285 case Builtin::BI__xray_customevent: {
5298 auto FTy = F->getFunctionType();
5299 auto Arg0 = E->
getArg(0);
5301 auto Arg0Ty = Arg0->
getType();
5302 auto PTy0 = FTy->getParamType(0);
5303 if (PTy0 != Arg0Val->getType()) {
5304 if (Arg0Ty->isArrayType())
5307 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
5310 auto PTy1 = FTy->getParamType(1);
5312 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
5313 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
5316 case Builtin::BI__xray_typedevent: {
5332 auto FTy = F->getFunctionType();
5334 auto PTy0 = FTy->getParamType(0);
5336 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
5337 auto Arg1 = E->
getArg(1);
5339 auto Arg1Ty = Arg1->
getType();
5340 auto PTy1 = FTy->getParamType(1);
5341 if (PTy1 != Arg1Val->getType()) {
5342 if (Arg1Ty->isArrayType())
5345 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
5348 auto PTy2 = FTy->getParamType(2);
5350 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
5351 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
5354 case Builtin::BI__builtin_ms_va_start:
5355 case Builtin::BI__builtin_ms_va_end:
5358 BuiltinID == Builtin::BI__builtin_ms_va_start));
5360 case Builtin::BI__builtin_ms_va_copy: {
5377 Value *ArgPtr = Builder.CreateLoad(SrcAddr,
"ap.val");
5378 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
5381 case Builtin::BI__builtin_get_device_side_mangled_name: {
5385 llvm::Constant *Zeros[] = {llvm::ConstantInt::get(
SizeTy, 0),
5386 llvm::ConstantInt::get(
SizeTy, 0)};
5387 auto *Ptr = llvm::ConstantExpr::getGetElementPtr(Str.getElementType(),
5388 Str.getPointer(), Zeros);
5414 LargestVectorWidth = std::max(LargestVectorWidth, VectorWidth);
5418 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
5420 llvm::Triple::getArchTypePrefix(
getTarget().getTriple().getArch());
5421 if (!Prefix.empty()) {
5422 IntrinsicID = Intrinsic::getIntrinsicForClangBuiltin(Prefix.data(), Name);
5426 if (IntrinsicID == Intrinsic::not_intrinsic)
5427 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
5430 if (IntrinsicID != Intrinsic::not_intrinsic) {
5435 unsigned ICEArguments = 0;
5441 llvm::FunctionType *FTy = F->getFunctionType();
5443 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
5446 if ((ICEArguments & (1 << i)) == 0) {
5451 ArgValue = llvm::ConstantInt::get(
5458 llvm::Type *PTy = FTy->getParamType(i);
5459 if (PTy != ArgValue->
getType()) {
5461 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
5462 if (PtrTy->getAddressSpace() !=
5463 ArgValue->
getType()->getPointerAddressSpace()) {
5464 ArgValue = Builder.CreateAddrSpaceCast(
5466 PtrTy->getAddressSpace()));
5470 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
5471 "Must be able to losslessly bit cast to param");
5474 if (PTy->isX86_AMXTy())
5475 ArgValue = Builder.CreateIntrinsic(Intrinsic::x86_cast_vector_to_tile,
5476 {ArgValue->
getType()}, {ArgValue});
5478 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
5481 Args.push_back(ArgValue);
5484 Value *
V = Builder.CreateCall(F, Args);
5487 llvm::Type *RetTy =
VoidTy;
5491 if (RetTy !=
V->getType()) {
5493 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
5494 if (PtrTy->getAddressSpace() !=
V->getType()->getPointerAddressSpace()) {
5495 V = Builder.CreateAddrSpaceCast(
5497 PtrTy->getAddressSpace()));
5501 assert(
V->getType()->canLosslesslyBitCastTo(RetTy) &&
5502 "Must be able to losslessly bit cast result type");
5505 if (
V->getType()->isX86_AMXTy())
5506 V = Builder.CreateIntrinsic(Intrinsic::x86_cast_tile_to_vector, {RetTy},
5509 V = Builder.CreateBitCast(
V, RetTy);
5512 if (RetTy->isVoidTy())
5532 if (
V->getType()->isVoidTy())
5539 llvm_unreachable(
"No current target builtin returns complex");
5541 llvm_unreachable(
"Bad evaluation kind in EmitBuiltinExpr");
5551 unsigned BuiltinID,
const CallExpr *E,
5553 llvm::Triple::ArchType Arch) {
5555 case llvm::Triple::arm:
5556 case llvm::Triple::armeb:
5557 case llvm::Triple::thumb:
5558 case llvm::Triple::thumbeb:
5560 case llvm::Triple::aarch64:
5561 case llvm::Triple::aarch64_32:
5562 case llvm::Triple::aarch64_be:
5564 case llvm::Triple::bpfeb:
5565 case llvm::Triple::bpfel:
5567 case llvm::Triple::x86:
5568 case llvm::Triple::x86_64:
5570 case llvm::Triple::ppc:
5571 case llvm::Triple::ppcle:
5572 case llvm::Triple::ppc64:
5573 case llvm::Triple::ppc64le:
5575 case llvm::Triple::r600:
5576 case llvm::Triple::amdgcn:
5578 case llvm::Triple::systemz:
5580 case llvm::Triple::nvptx:
5581 case llvm::Triple::nvptx64:
5583 case llvm::Triple::wasm32:
5584 case llvm::Triple::wasm64:
5586 case llvm::Triple::hexagon:
5588 case llvm::Triple::riscv32:
5589 case llvm::Triple::riscv64:
5591 case llvm::Triple::loongarch32:
5592 case llvm::Triple::loongarch64:
5603 assert(
getContext().getAuxTargetInfo() &&
"Missing aux target info");
5615 bool HasLegalHalfType =
true,
5617 bool AllowBFloatArgsAndRet =
true) {
5618 int IsQuad = TypeFlags.
isQuad();
5622 return llvm::FixedVectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
5625 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5627 if (AllowBFloatArgsAndRet)
5628 return llvm::FixedVectorType::get(CGF->
BFloatTy, V1Ty ? 1 : (4 << IsQuad));
5630 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5632 if (HasLegalHalfType)
5633 return llvm::FixedVectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
5635 return llvm::FixedVectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
5637 return llvm::FixedVectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
5640 return llvm::FixedVectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
5645 return llvm::FixedVectorType::get(CGF->
Int8Ty, 16);
5647 return llvm::FixedVectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
5649 return llvm::FixedVectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
5651 llvm_unreachable(
"Unknown vector element type!");
5656 int IsQuad = IntTypeFlags.
isQuad();
5659 return llvm::FixedVectorType::get(CGF->
HalfTy, (4 << IsQuad));
5661 return llvm::FixedVectorType::get(CGF->
FloatTy, (2 << IsQuad));
5663 return llvm::FixedVectorType::get(CGF->
DoubleTy, (1 << IsQuad));
5665 llvm_unreachable(
"Type can't be converted to floating-point!");
5670 const ElementCount &Count) {
5671 Value *SV = llvm::ConstantVector::getSplat(Count,
C);
5672 return Builder.CreateShuffleVector(
V,
V, SV,
"lane");
5682 unsigned shift,
bool rightshift) {
5684 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
5685 ai != ae; ++ai, ++j) {
5686 if (F->isConstrainedFPIntrinsic())
5687 if (ai->getType()->isMetadataTy())
5689 if (shift > 0 && shift == j)
5692 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
5695 if (F->isConstrainedFPIntrinsic())
5696 return Builder.CreateConstrainedFPCall(F, Ops, name);
5698 return Builder.CreateCall(F, Ops, name);
5704 return ConstantInt::get(Ty, neg ? -SV : SV);
5709 llvm::Type *Ty,
bool usgn,
5714 int EltSize = VTy->getScalarSizeInBits();
5716 Vec = Builder.CreateBitCast(Vec, Ty);
5720 if (ShiftAmt == EltSize) {
5723 return llvm::ConstantAggregateZero::get(VTy);
5728 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
5734 return Builder.CreateLShr(Vec, Shift, name);
5736 return Builder.CreateAShr(Vec, Shift, name);
5762struct ARMVectorIntrinsicInfo {
5763 const char *NameHint;
5765 unsigned LLVMIntrinsic;
5766 unsigned AltLLVMIntrinsic;
5769 bool operator<(
unsigned RHSBuiltinID)
const {
5770 return BuiltinID < RHSBuiltinID;
5772 bool operator<(
const ARMVectorIntrinsicInfo &TE)
const {
5773 return BuiltinID < TE.BuiltinID;
5778#define NEONMAP0(NameBase) \
5779 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 }
5781#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
5782 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
5783 Intrinsic::LLVMIntrinsic, 0, TypeModifier }
5785#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \
5786 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \
5787 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \
5791 NEONMAP1(__a32_vcvt_bf16_f32, arm_neon_vcvtfp2bf, 0),
5798 NEONMAP1(vabs_v, arm_neon_vabs, 0),
5799 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
5803 NEONMAP1(vaesdq_u8, arm_neon_aesd, 0),
5804 NEONMAP1(vaeseq_u8, arm_neon_aese, 0),
5805 NEONMAP1(vaesimcq_u8, arm_neon_aesimc, 0),
5806 NEONMAP1(vaesmcq_u8, arm_neon_aesmc, 0),
5807 NEONMAP1(vbfdot_f32, arm_neon_bfdot, 0),
5808 NEONMAP1(vbfdotq_f32, arm_neon_bfdot, 0),
5809 NEONMAP1(vbfmlalbq_f32, arm_neon_bfmlalb, 0),
5810 NEONMAP1(vbfmlaltq_f32, arm_neon_bfmlalt, 0),
5811 NEONMAP1(vbfmmlaq_f32, arm_neon_bfmmla, 0),
5824 NEONMAP1(vcage_v, arm_neon_vacge, 0),
5825 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
5826 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
5827 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
5828 NEONMAP1(vcale_v, arm_neon_vacge, 0),
5829 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
5830 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
5831 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
5848 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
5851 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
5853 NEONMAP1(vcvt_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
5854 NEONMAP1(vcvt_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
5855 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5856 NEONMAP1(vcvt_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
5857 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
5858 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
5859 NEONMAP1(vcvt_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
5860 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
5861 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
5868 NEONMAP1(vcvta_s16_f16, arm_neon_vcvtas, 0),
5869 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
5870 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
5871 NEONMAP1(vcvta_u16_f16, arm_neon_vcvtau, 0),
5872 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
5873 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
5874 NEONMAP1(vcvtaq_s16_f16, arm_neon_vcvtas, 0),
5875 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
5876 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
5877 NEONMAP1(vcvtaq_u16_f16, arm_neon_vcvtau, 0),
5878 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
5879 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
5880 NEONMAP1(vcvth_bf16_f32, arm_neon_vcvtbfp2bf, 0),
5881 NEONMAP1(vcvtm_s16_f16, arm_neon_vcvtms, 0),
5882 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
5883 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
5884 NEONMAP1(vcvtm_u16_f16, arm_neon_vcvtmu, 0),
5885 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
5886 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
5887 NEONMAP1(vcvtmq_s16_f16, arm_neon_vcvtms, 0),
5888 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
5889 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
5890 NEONMAP1(vcvtmq_u16_f16, arm_neon_vcvtmu, 0),
5891 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
5892 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
5893 NEONMAP1(vcvtn_s16_f16, arm_neon_vcvtns, 0),
5894 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
5895 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
5896 NEONMAP1(vcvtn_u16_f16, arm_neon_vcvtnu, 0),
5897 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
5898 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
5899 NEONMAP1(vcvtnq_s16_f16, arm_neon_vcvtns, 0),
5900 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
5901 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
5902 NEONMAP1(vcvtnq_u16_f16, arm_neon_vcvtnu, 0),
5903 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
5904 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
5905 NEONMAP1(vcvtp_s16_f16, arm_neon_vcvtps, 0),
5906 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
5907 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
5908 NEONMAP1(vcvtp_u16_f16, arm_neon_vcvtpu, 0),
5909 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
5910 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
5911 NEONMAP1(vcvtpq_s16_f16, arm_neon_vcvtps, 0),
5912 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
5913 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
5914 NEONMAP1(vcvtpq_u16_f16, arm_neon_vcvtpu, 0),
5915 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
5916 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
5920 NEONMAP1(vcvtq_n_f16_s16, arm_neon_vcvtfxs2fp, 0),
5921 NEONMAP1(vcvtq_n_f16_u16, arm_neon_vcvtfxu2fp, 0),
5922 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
5923 NEONMAP1(vcvtq_n_s16_f16, arm_neon_vcvtfp2fxs, 0),
5924 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
5925 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
5926 NEONMAP1(vcvtq_n_u16_f16, arm_neon_vcvtfp2fxu, 0),
5927 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
5928 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
5935 NEONMAP1(vdot_s32, arm_neon_sdot, 0),
5936 NEONMAP1(vdot_u32, arm_neon_udot, 0),
5937 NEONMAP1(vdotq_s32, arm_neon_sdot, 0),
5938 NEONMAP1(vdotq_u32, arm_neon_udot, 0),
5948 NEONMAP1(vld1_v, arm_neon_vld1, 0),
5949 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
5950 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
5951 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
5953 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
5954 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
5955 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
5956 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
5957 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
5958 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
5959 NEONMAP1(vld2_v, arm_neon_vld2, 0),
5960 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
5961 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
5962 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
5963 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
5964 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
5965 NEONMAP1(vld3_v, arm_neon_vld3, 0),
5966 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
5967 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
5968 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
5969 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
5970 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
5971 NEONMAP1(vld4_v, arm_neon_vld4, 0),
5972 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
5973 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
5974 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
5983 NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
5984 NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
6002 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, sadd_sat, 0),
6003 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, ssub_sat, 0),
6027 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
6028 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
6032 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6033 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
6056 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6057 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
6061 NEONMAP1(vsha1su0q_u32, arm_neon_sha1su0, 0),
6062 NEONMAP1(vsha1su1q_u32, arm_neon_sha1su1, 0),
6063 NEONMAP1(vsha256h2q_u32, arm_neon_sha256h2, 0),
6064 NEONMAP1(vsha256hq_u32, arm_neon_sha256h, 0),
6065 NEONMAP1(vsha256su0q_u32, arm_neon_sha256su0, 0),
6066 NEONMAP1(vsha256su1q_u32, arm_neon_sha256su1, 0),
6075 NEONMAP1(vst1_v, arm_neon_vst1, 0),
6076 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
6077 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
6078 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
6079 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
6080 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
6081 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
6082 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
6083 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
6084 NEONMAP1(vst2_v, arm_neon_vst2, 0),
6085 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
6086 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
6087 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
6088 NEONMAP1(vst3_v, arm_neon_vst3, 0),
6089 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
6090 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
6091 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
6092 NEONMAP1(vst4_v, arm_neon_vst4, 0),
6093 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
6094 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
6100 NEONMAP1(vusdot_s32, arm_neon_usdot, 0),
6101 NEONMAP1(vusdotq_s32, arm_neon_usdot, 0),
6102 NEONMAP1(vusmmlaq_s32, arm_neon_usmmla, 0),
6110 NEONMAP1(__a64_vcvtq_low_bf16_f32, aarch64_neon_bfcvtn, 0),
6115 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
6116 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
6121 NEONMAP1(vaesdq_u8, aarch64_crypto_aesd, 0),
6122 NEONMAP1(vaeseq_u8, aarch64_crypto_aese, 0),
6123 NEONMAP1(vaesimcq_u8, aarch64_crypto_aesimc, 0),
6124 NEONMAP1(vaesmcq_u8, aarch64_crypto_aesmc, 0),
6133 NEONMAP1(vbfdot_f32, aarch64_neon_bfdot, 0),
6134 NEONMAP1(vbfdotq_f32, aarch64_neon_bfdot, 0),
6135 NEONMAP1(vbfmlalbq_f32, aarch64_neon_bfmlalb, 0),
6136 NEONMAP1(vbfmlaltq_f32, aarch64_neon_bfmlalt, 0),
6137 NEONMAP1(vbfmmlaq_f32, aarch64_neon_bfmmla, 0),
6148 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
6149 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
6150 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
6151 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
6152 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
6153 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
6154 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
6155 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
6192 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
6195 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
6197 NEONMAP1(vcvt_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6198 NEONMAP1(vcvt_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6199 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6200 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6201 NEONMAP1(vcvt_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6202 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6203 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6204 NEONMAP1(vcvt_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6205 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6206 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6210 NEONMAP1(vcvtq_high_bf16_f32, aarch64_neon_bfcvtn2, 0),
6211 NEONMAP1(vcvtq_n_f16_s16, aarch64_neon_vcvtfxs2fp, 0),
6212 NEONMAP1(vcvtq_n_f16_u16, aarch64_neon_vcvtfxu2fp, 0),
6213 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6214 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
6215 NEONMAP1(vcvtq_n_s16_f16, aarch64_neon_vcvtfp2fxs, 0),
6216 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
6217 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
6218 NEONMAP1(vcvtq_n_u16_f16, aarch64_neon_vcvtfp2fxu, 0),
6219 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
6220 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
6222 NEONMAP1(vdot_s32, aarch64_neon_sdot, 0),
6223 NEONMAP1(vdot_u32, aarch64_neon_udot, 0),
6224 NEONMAP1(vdotq_s32, aarch64_neon_sdot, 0),
6225 NEONMAP1(vdotq_u32, aarch64_neon_udot, 0),
6238 NEONMAP1(vfmlal_high_f16, aarch64_neon_fmlal2, 0),
6239 NEONMAP1(vfmlal_low_f16, aarch64_neon_fmlal, 0),
6240 NEONMAP1(vfmlalq_high_f16, aarch64_neon_fmlal2, 0),
6241 NEONMAP1(vfmlalq_low_f16, aarch64_neon_fmlal, 0),
6242 NEONMAP1(vfmlsl_high_f16, aarch64_neon_fmlsl2, 0),
6243 NEONMAP1(vfmlsl_low_f16, aarch64_neon_fmlsl, 0),
6244 NEONMAP1(vfmlslq_high_f16, aarch64_neon_fmlsl2, 0),
6245 NEONMAP1(vfmlslq_low_f16, aarch64_neon_fmlsl, 0),
6250 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
6251 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
6252 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
6253 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
6254 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
6255 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
6256 NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
6257 NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
6270 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
6271 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
6272 NEONMAP1(vqdmulh_lane_v, aarch64_neon_sqdmulh_lane, 0),
6273 NEONMAP1(vqdmulh_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6275 NEONMAP1(vqdmulhq_lane_v, aarch64_neon_sqdmulh_lane, 0),
6276 NEONMAP1(vqdmulhq_laneq_v, aarch64_neon_sqdmulh_laneq, 0),
6291 NEONMAP1(vqrdmulh_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6292 NEONMAP1(vqrdmulh_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6294 NEONMAP1(vqrdmulhq_lane_v, aarch64_neon_sqrdmulh_lane, 0),
6295 NEONMAP1(vqrdmulhq_laneq_v, aarch64_neon_sqrdmulh_laneq, 0),
6303 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
6304 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
6308 NEONMAP1(vrax1q_u64, aarch64_crypto_rax1, 0),
6309 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6310 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
6329 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6330 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
6334 NEONMAP1(vsha1su0q_u32, aarch64_crypto_sha1su0, 0),
6335 NEONMAP1(vsha1su1q_u32, aarch64_crypto_sha1su1, 0),
6336 NEONMAP1(vsha256h2q_u32, aarch64_crypto_sha256h2, 0),
6337 NEONMAP1(vsha256hq_u32, aarch64_crypto_sha256h, 0),
6338 NEONMAP1(vsha256su0q_u32, aarch64_crypto_sha256su0, 0),
6339 NEONMAP1(vsha256su1q_u32, aarch64_crypto_sha256su1, 0),
6340 NEONMAP1(vsha512h2q_u64, aarch64_crypto_sha512h2, 0),
6341 NEONMAP1(vsha512hq_u64, aarch64_crypto_sha512h, 0),
6342 NEONMAP1(vsha512su0q_u64, aarch64_crypto_sha512su0, 0),
6343 NEONMAP1(vsha512su1q_u64, aarch64_crypto_sha512su1, 0),
6352 NEONMAP1(vsm3partw1q_u32, aarch64_crypto_sm3partw1, 0),
6353 NEONMAP1(vsm3partw2q_u32, aarch64_crypto_sm3partw2, 0),
6354 NEONMAP1(vsm3ss1q_u32, aarch64_crypto_sm3ss1, 0),
6355 NEONMAP1(vsm3tt1aq_u32, aarch64_crypto_sm3tt1a, 0),
6356 NEONMAP1(vsm3tt1bq_u32, aarch64_crypto_sm3tt1b, 0),
6357 NEONMAP1(vsm3tt2aq_u32, aarch64_crypto_sm3tt2a, 0),
6358 NEONMAP1(vsm3tt2bq_u32, aarch64_crypto_sm3tt2b, 0),
6359 NEONMAP1(vsm4ekeyq_u32, aarch64_crypto_sm4ekey, 0),
6360 NEONMAP1(vsm4eq_u32, aarch64_crypto_sm4e, 0),
6361 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
6362 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
6363 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
6364 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
6365 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
6366 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
6370 NEONMAP1(vusdot_s32, aarch64_neon_usdot, 0),
6371 NEONMAP1(vusdotq_s32, aarch64_neon_usdot, 0),
6372 NEONMAP1(vusmmlaq_s32, aarch64_neon_usmmla, 0),
6373 NEONMAP1(vxarq_u64, aarch64_crypto_xar, 0),
6411 NEONMAP1(vcvth_bf16_f32, aarch64_neon_bfcvt, 0),
6430 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
6451 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
6479 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
6560 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
6561 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
6562 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
6563 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
6617 { NEON::BI__builtin_neon_splat_lane_bf16, NEON::BI__builtin_neon_splat_lane_v, },
6618 { NEON::BI__builtin_neon_splat_laneq_bf16, NEON::BI__builtin_neon_splat_laneq_v, },
6619 { NEON::BI__builtin_neon_splatq_lane_bf16, NEON::BI__builtin_neon_splatq_lane_v, },
6620 { NEON::BI__builtin_neon_splatq_laneq_bf16, NEON::BI__builtin_neon_splatq_laneq_v, },
6621 { NEON::BI__builtin_neon_vabd_f16, NEON::BI__builtin_neon_vabd_v, },
6622 { NEON::BI__builtin_neon_vabdq_f16, NEON::BI__builtin_neon_vabdq_v, },
6623 { NEON::BI__builtin_neon_vabs_f16, NEON::BI__builtin_neon_vabs_v, },
6624 { NEON::BI__builtin_neon_vabsq_f16, NEON::BI__builtin_neon_vabsq_v, },
6625 { NEON::BI__builtin_neon_vbsl_f16, NEON::BI__builtin_neon_vbsl_v, },
6626 { NEON::BI__builtin_neon_vbslq_f16, NEON::BI__builtin_neon_vbslq_v, },
6627 { NEON::BI__builtin_neon_vcage_f16, NEON::BI__builtin_neon_vcage_v, },
6628 { NEON::BI__builtin_neon_vcageq_f16, NEON::BI__builtin_neon_vcageq_v, },
6629 { NEON::BI__builtin_neon_vcagt_f16, NEON::BI__builtin_neon_vcagt_v, },
6630 { NEON::BI__builtin_neon_vcagtq_f16, NEON::BI__builtin_neon_vcagtq_v, },
6631 { NEON::BI__builtin_neon_vcale_f16, NEON::BI__builtin_neon_vcale_v, },
6632 { NEON::BI__builtin_neon_vcaleq_f16, NEON::BI__builtin_neon_vcaleq_v, },
6633 { NEON::BI__builtin_neon_vcalt_f16, NEON::BI__builtin_neon_vcalt_v, },
6634 { NEON::BI__builtin_neon_vcaltq_f16, NEON::BI__builtin_neon_vcaltq_v, },
6635 { NEON::BI__builtin_neon_vceqz_f16, NEON::BI__builtin_neon_vceqz_v, },
6636 { NEON::BI__builtin_neon_vceqzq_f16, NEON::BI__builtin_neon_vceqzq_v, },
6637 { NEON::BI__builtin_neon_vcgez_f16, NEON::BI__builtin_neon_vcgez_v, },
6638 { NEON::BI__builtin_neon_vcgezq_f16, NEON::BI__builtin_neon_vcgezq_v, },
6639 { NEON::BI__builtin_neon_vcgtz_f16, NEON::BI__builtin_neon_vcgtz_v, },
6640 { NEON::BI__builtin_neon_vcgtzq_f16, NEON::BI__builtin_neon_vcgtzq_v, },
6641 { NEON::BI__builtin_neon_vclez_f16, NEON::BI__builtin_neon_vclez_v, },
6642 { NEON::BI__builtin_neon_vclezq_f16, NEON::BI__builtin_neon_vclezq_v, },
6643 { NEON::BI__builtin_neon_vcltz_f16, NEON::BI__builtin_neon_vcltz_v, },
6644 { NEON::BI__builtin_neon_vcltzq_f16, NEON::BI__builtin_neon_vcltzq_v, },
6645 { NEON::BI__builtin_neon_vext_f16, NEON::BI__builtin_neon_vext_v, },
6646 { NEON::BI__builtin_neon_vextq_f16, NEON::BI__builtin_neon_vextq_v, },
6647 { NEON::BI__builtin_neon_vfma_f16, NEON::BI__builtin_neon_vfma_v, },
6648 { NEON::BI__builtin_neon_vfma_lane_f16, NEON::BI__builtin_neon_vfma_lane_v, },
6649 { NEON::BI__builtin_neon_vfma_laneq_f16, NEON::BI__builtin_neon_vfma_laneq_v, },
6650 { NEON::BI__builtin_neon_vfmaq_f16, NEON::BI__builtin_neon_vfmaq_v, },
6651 { NEON::BI__builtin_neon_vfmaq_lane_f16, NEON::BI__builtin_neon_vfmaq_lane_v, },
6652 { NEON::BI__builtin_neon_vfmaq_laneq_f16, NEON::BI__builtin_neon_vfmaq_laneq_v, },
6653 { NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v },
6654 { NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v },
6655 { NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v },
6656 { NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v },
6657 { NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v },
6658 { NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v },
6659 { NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v },
6660 { NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v },
6661 { NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v },
6662 { NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v },
6663 { NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v },
6664 { NEON::BI__builtin_neon_vld1q_lane_bf16, NEON::BI__builtin_neon_vld1q_lane_v },
6665 { NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v },
6666 { NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v },
6667 { NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v },
6668 { NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v },
6669 { NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v },
6670 { NEON::BI__builtin_neon_vld2q_lane_bf16, NEON::BI__builtin_neon_vld2q_lane_v },
6671 { NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v },
6672 { NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v },
6673 { NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v },
6674 { NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v },
6675 { NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v },
6676 { NEON::BI__builtin_neon_vld3q_lane_bf16, NEON::BI__builtin_neon_vld3q_lane_v },
6677 { NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v },
6678 { NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v },
6679 { NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v },
6680 { NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v },
6681 { NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v },
6682 { NEON::BI__builtin_neon_vld4q_lane_bf16, NEON::BI__builtin_neon_vld4q_lane_v },
6683 { NEON::BI__builtin_neon_vmax_f16, NEON::BI__builtin_neon_vmax_v, },
6684 { NEON::BI__builtin_neon_vmaxnm_f16, NEON::BI__builtin_neon_vmaxnm_v, },
6685 { NEON::BI__builtin_neon_vmaxnmq_f16, NEON::BI__builtin_neon_vmaxnmq_v, },
6686 { NEON::BI__builtin_neon_vmaxq_f16, NEON::BI__builtin_neon_vmaxq_v, },
6687 { NEON::BI__builtin_neon_vmin_f16, NEON::BI__builtin_neon_vmin_v, },
6688 { NEON::BI__builtin_neon_vminnm_f16, NEON::BI__builtin_neon_vminnm_v, },
6689 { NEON::BI__builtin_neon_vminnmq_f16, NEON::BI__builtin_neon_vminnmq_v, },
6690 { NEON::BI__builtin_neon_vminq_f16, NEON::BI__builtin_neon_vminq_v, },
6691 { NEON::BI__builtin_neon_vmulx_f16, NEON::BI__builtin_neon_vmulx_v, },
6692 { NEON::BI__builtin_neon_vmulxq_f16, NEON::BI__builtin_neon_vmulxq_v, },
6693 { NEON::BI__builtin_neon_vpadd_f16, NEON::BI__builtin_neon_vpadd_v, },
6694 { NEON::BI__builtin_neon_vpaddq_f16, NEON::BI__builtin_neon_vpaddq_v, },
6695 { NEON::BI__builtin_neon_vpmax_f16, NEON::BI__builtin_neon_vpmax_v, },
6696 { NEON::BI__builtin_neon_vpmaxnm_f16, NEON::BI__builtin_neon_vpmaxnm_v, },
6697 { NEON::BI__builtin_neon_vpmaxnmq_f16, NEON::BI__builtin_neon_vpmaxnmq_v, },
6698 { NEON::BI__builtin_neon_vpmaxq_f16, NEON::BI__builtin_neon_vpmaxq_v, },
6699 { NEON::BI__builtin_neon_vpmin_f16, NEON::BI__builtin_neon_vpmin_v, },
6700 { NEON::BI__builtin_neon_vpminnm_f16, NEON::BI__builtin_neon_vpminnm_v, },
6701 { NEON::BI__builtin_neon_vpminnmq_f16, NEON::BI__builtin_neon_vpminnmq_v, },
6702 { NEON::BI__builtin_neon_vpminq_f16, NEON::BI__builtin_neon_vpminq_v, },
6703 { NEON::BI__builtin_neon_vrecpe_f16, NEON::BI__builtin_neon_vrecpe_v, },
6704 { NEON::BI__builtin_neon_vrecpeq_f16, NEON::BI__builtin_neon_vrecpeq_v, },
6705 { NEON::BI__builtin_neon_vrecps_f16, NEON::BI__builtin_neon_vrecps_v, },
6706 { NEON::BI__builtin_neon_vrecpsq_f16, NEON::BI__builtin_neon_vrecpsq_v, },
6707 { NEON::BI__builtin_neon_vrnd_f16, NEON::BI__builtin_neon_vrnd_v, },
6708 { NEON::BI__builtin_neon_vrnda_f16, NEON::BI__builtin_neon_vrnda_v, },
6709 { NEON::BI__builtin_neon_vrndaq_f16, NEON::BI__builtin_neon_vrndaq_v, },
6710 { NEON::BI__builtin_neon_vrndi_f16, NEON::BI__builtin_neon_vrndi_v, },
6711 { NEON::BI__builtin_neon_vrndiq_f16, NEON::BI__builtin_neon_vrndiq_v, },
6712 { NEON::BI__builtin_neon_vrndm_f16, NEON::BI__builtin_neon_vrndm_v, },
6713 { NEON::BI__builtin_neon_vrndmq_f16, NEON::BI__builtin_neon_vrndmq_v, },
6714 { NEON::BI__builtin_neon_vrndn_f16, NEON::BI__builtin_neon_vrndn_v, },
6715 { NEON::BI__builtin_neon_vrndnq_f16, NEON::BI__builtin_neon_vrndnq_v, },
6716 { NEON::BI__builtin_neon_vrndp_f16, NEON::BI__builtin_neon_vrndp_v, },
6717 { NEON::BI__builtin_neon_vrndpq_f16, NEON::BI__builtin_neon_vrndpq_v, },
6718 { NEON::BI__builtin_neon_vrndq_f16, NEON::BI__builtin_neon_vrndq_v, },
6719 { NEON::BI__builtin_neon_vrndx_f16, NEON::BI__builtin_neon_vrndx_v, },
6720 { NEON::BI__builtin_neon_vrndxq_f16, NEON::BI__builtin_neon_vrndxq_v, },
6721 { NEON::BI__builtin_neon_vrsqrte_f16, NEON::BI__builtin_neon_vrsqrte_v, },
6722 { NEON::BI__builtin_neon_vrsqrteq_f16, NEON::BI__builtin_neon_vrsqrteq_v, },
6723 { NEON::BI__builtin_neon_vrsqrts_f16, NEON::BI__builtin_neon_vrsqrts_v, },
6724 { NEON::BI__builtin_neon_vrsqrtsq_f16, NEON::BI__builtin_neon_vrsqrtsq_v, },
6725 { NEON::BI__builtin_neon_vsqrt_f16, NEON::BI__builtin_neon_vsqrt_v, },
6726 { NEON::BI__builtin_neon_vsqrtq_f16, NEON::BI__builtin_neon_vsqrtq_v, },
6727 { NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v },
6728 { NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v },
6729 { NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v },
6730 { NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v },
6731 { NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v },
6732 { NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v },
6733 { NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v },
6734 { NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v },
6735 { NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v },
6736 { NEON::BI__builtin_neon_vst1q_lane_bf16, NEON::BI__builtin_neon_vst1q_lane_v },
6737 { NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v },
6738 { NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v },
6739 { NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v },
6740 { NEON::BI__builtin_neon_vst2q_lane_bf16, NEON::BI__builtin_neon_vst2q_lane_v },
6741 { NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v },
6742 { NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v },
6743 { NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v },
6744 { NEON::BI__builtin_neon_vst3q_lane_bf16, NEON::BI__builtin_neon_vst3q_lane_v },
6745 { NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v },
6746 { NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v },
6747 { NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v },
6748 { NEON::BI__builtin_neon_vst4q_lane_bf16, NEON::BI__builtin_neon_vst4q_lane_v },
6749 { NEON::BI__builtin_neon_vtrn_f16, NEON::BI__builtin_neon_vtrn_v, },
6750 { NEON::BI__builtin_neon_vtrnq_f16, NEON::BI__builtin_neon_vtrnq_v, },
6751 { NEON::BI__builtin_neon_vuzp_f16, NEON::BI__builtin_neon_vuzp_v, },
6752 { NEON::BI__builtin_neon_vuzpq_f16, NEON::BI__builtin_neon_vuzpq_v, },
6753 { NEON::BI__builtin_neon_vzip_f16, NEON::BI__builtin_neon_vzip_v, },
6754 { NEON::BI__builtin_neon_vzipq_f16, NEON::BI__builtin_neon_vzipq_v, },
6758 { NEON::BI__builtin_neon_vldap1_lane_u64, NEON::BI__builtin_neon_vldap1_lane_s64 },
6759 { NEON::BI__builtin_neon_vldap1_lane_f64, NEON::BI__builtin_neon_vldap1_lane_s64 },
6760 { NEON::BI__builtin_neon_vldap1_lane_p64, NEON::BI__builtin_neon_vldap1_lane_s64 },
6761 { NEON::BI__builtin_neon_vldap1q_lane_u64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
6762 { NEON::BI__builtin_neon_vldap1q_lane_f64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
6763 { NEON::BI__builtin_neon_vldap1q_lane_p64, NEON::BI__builtin_neon_vldap1q_lane_s64 },
6764 { NEON::BI__builtin_neon_vstl1_lane_u64, NEON::BI__builtin_neon_vstl1_lane_s64 },
6765 { NEON::BI__builtin_neon_vstl1_lane_f64, NEON::BI__builtin_neon_vstl1_lane_s64 },
6766 { NEON::BI__builtin_neon_vstl1_lane_p64, NEON::BI__builtin_neon_vstl1_lane_s64 },
6767 { NEON::BI__builtin_neon_vstl1q_lane_u64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
6768 { NEON::BI__builtin_neon_vstl1q_lane_f64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
6769 { NEON::BI__builtin_neon_vstl1q_lane_p64, NEON::BI__builtin_neon_vstl1q_lane_s64 },
6776#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6778 #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
6782#define SVEMAP2(NameBase, TypeModifier) \
6783 { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier }
6785#define GET_SVE_LLVM_INTRINSIC_MAP
6786#include "clang/Basic/arm_sve_builtin_cg.inc"
6787#include "clang/Basic/BuiltinsAArch64NeonSVEBridge_cg.def"
6788#undef GET_SVE_LLVM_INTRINSIC_MAP
6794#define SMEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \
6796 #NameBase, SME::BI__builtin_sme_##NameBase, Intrinsic::LLVMIntrinsic, 0, \
6800#define SMEMAP2(NameBase, TypeModifier) \
6801 { #NameBase, SME::BI__builtin_sme_##NameBase, 0, 0, TypeModifier }
6803#define GET_SME_LLVM_INTRINSIC_MAP
6804#include "clang/Basic/arm_sme_builtin_cg.inc"
6805#undef GET_SME_LLVM_INTRINSIC_MAP
6818static const ARMVectorIntrinsicInfo *
6820 unsigned BuiltinID,
bool &MapProvenSorted) {
6823 if (!MapProvenSorted) {
6824 assert(llvm::is_sorted(IntrinsicMap));
6825 MapProvenSorted =
true;
6829 const ARMVectorIntrinsicInfo *Builtin =
6830 llvm::lower_bound(IntrinsicMap, BuiltinID);
6832 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
6840 llvm::Type *ArgType,
6853 Ty = llvm::FixedVectorType::get(
6854 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
6861 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
6862 ArgType = llvm::FixedVectorType::get(ArgType, Elts);
6866 Tys.push_back(ArgType);
6869 Tys.push_back(ArgType);
6880 unsigned BuiltinID = SISDInfo.BuiltinID;
6881 unsigned int Int = SISDInfo.LLVMIntrinsic;
6882 unsigned Modifier = SISDInfo.TypeModifier;
6883 const char *
s = SISDInfo.NameHint;
6885 switch (BuiltinID) {
6886 case NEON::BI__builtin_neon_vcled_s64:
6887 case NEON::BI__builtin_neon_vcled_u64:
6888 case NEON::BI__builtin_neon_vcles_f32:
6889 case NEON::BI__builtin_neon_vcled_f64:
6890 case NEON::BI__builtin_neon_vcltd_s64:
6891 case NEON::BI__builtin_neon_vcltd_u64:
6892 case NEON::BI__builtin_neon_vclts_f32:
6893 case NEON::BI__builtin_neon_vcltd_f64:
6894 case NEON::BI__builtin_neon_vcales_f32:
6895 case NEON::BI__builtin_neon_vcaled_f64:
6896 case NEON::BI__builtin_neon_vcalts_f32:
6897 case NEON::BI__builtin_neon_vcaltd_f64:
6901 std::swap(Ops[0], Ops[1]);
6905 assert(Int &&
"Generic code assumes a valid intrinsic");
6913 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
6914 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
6915 ai != ae; ++ai, ++j) {
6916 llvm::Type *ArgTy = ai->getType();
6917 if (Ops[j]->
getType()->getPrimitiveSizeInBits() ==
6918 ArgTy->getPrimitiveSizeInBits())
6921 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
6924 Ops[j] = CGF.
Builder.CreateTruncOrBitCast(
6925 Ops[j], cast<llvm::VectorType>(ArgTy)->getElementType());
6927 CGF.
Builder.CreateInsertElement(PoisonValue::get(ArgTy), Ops[j], C0);
6932 if (ResultType->getPrimitiveSizeInBits().getFixedValue() <
6933 Result->getType()->getPrimitiveSizeInBits().getFixedValue())
6940 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
6941 const char *NameHint,
unsigned Modifier,
const CallExpr *E,
6943 llvm::Triple::ArchType Arch) {
6946 std::optional<llvm::APSInt> NeonTypeConst =
6953 bool Usgn =
Type.isUnsigned();
6954 bool Quad =
Type.isQuad();
6956 const bool AllowBFloatArgsAndRet =
6959 llvm::FixedVectorType *VTy =
6960 GetNeonType(
this,
Type, HasLegalHalfType,
false, AllowBFloatArgsAndRet);
6961 llvm::Type *Ty = VTy;
6965 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
6966 return Builder.getInt32(addr.getAlignment().getQuantity());
6969 unsigned Int = LLVMIntrinsic;
6971 Int = AltLLVMIntrinsic;
6973 switch (BuiltinID) {
6975 case NEON::BI__builtin_neon_splat_lane_v:
6976 case NEON::BI__builtin_neon_splat_laneq_v:
6977 case NEON::BI__builtin_neon_splatq_lane_v:
6978 case NEON::BI__builtin_neon_splatq_laneq_v: {
6979 auto NumElements = VTy->getElementCount();
6980 if (BuiltinID == NEON::BI__builtin_neon_splatq_lane_v)
6981 NumElements = NumElements * 2;
6982 if (BuiltinID == NEON::BI__builtin_neon_splat_laneq_v)
6983 NumElements = NumElements.divideCoefficientBy(2);
6985 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
6986 return EmitNeonSplat(Ops[0], cast<ConstantInt>(Ops[1]), NumElements);
6988 case NEON::BI__builtin_neon_vpadd_v:
6989 case NEON::BI__builtin_neon_vpaddq_v:
6991 if (VTy->getElementType()->isFloatingPointTy() &&
6992 Int == Intrinsic::aarch64_neon_addp)
6993 Int = Intrinsic::aarch64_neon_faddp;
6995 case NEON::BI__builtin_neon_vabs_v:
6996 case NEON::BI__builtin_neon_vabsq_v:
6997 if (VTy->getElementType()->isFloatingPointTy())
7000 case NEON::BI__builtin_neon_vadd_v:
7001 case NEON::BI__builtin_neon_vaddq_v: {
7002 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, Quad ? 16 : 8);
7003 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
7004 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
7005 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
7006 return Builder.CreateBitCast(Ops[0], Ty);
7008 case NEON::BI__builtin_neon_vaddhn_v: {
7009 llvm::FixedVectorType *SrcTy =
7010 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7013 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7014 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
7015 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
7018 Constant *ShiftAmt =
7019 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7020 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
7023 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
7025 case NEON::BI__builtin_neon_vcale_v:
7026 case NEON::BI__builtin_neon_vcaleq_v:
7027 case NEON::BI__builtin_neon_vcalt_v:
7028 case NEON::BI__builtin_neon_vcaltq_v:
7029 std::swap(Ops[0], Ops[1]);
7031 case NEON::BI__builtin_neon_vcage_v:
7032 case NEON::BI__builtin_neon_vcageq_v:
7033 case NEON::BI__builtin_neon_vcagt_v:
7034 case NEON::BI__builtin_neon_vcagtq_v: {
7036 switch (VTy->getScalarSizeInBits()) {
7037 default: llvm_unreachable(
"unexpected type");
7048 auto *VecFlt = llvm::FixedVectorType::get(Ty, VTy->getNumElements());
7049 llvm::Type *Tys[] = { VTy, VecFlt };
7053 case NEON::BI__builtin_neon_vceqz_v:
7054 case NEON::BI__builtin_neon_vceqzq_v:
7056 ICmpInst::ICMP_EQ,
"vceqz");
7057 case NEON::BI__builtin_neon_vcgez_v:
7058 case NEON::BI__builtin_neon_vcgezq_v:
7060 ICmpInst::ICMP_SGE,
"vcgez");
7061 case NEON::BI__builtin_neon_vclez_v:
7062 case NEON::BI__builtin_neon_vclezq_v:
7064 ICmpInst::ICMP_SLE,
"vclez");
7065 case NEON::BI__builtin_neon_vcgtz_v:
7066 case NEON::BI__builtin_neon_vcgtzq_v:
7068 ICmpInst::ICMP_SGT,
"vcgtz");
7069 case NEON::BI__builtin_neon_vcltz_v:
7070 case NEON::BI__builtin_neon_vcltzq_v:
7072 ICmpInst::ICMP_SLT,
"vcltz");
7073 case NEON::BI__builtin_neon_vclz_v:
7074 case NEON::BI__builtin_neon_vclzq_v:
7077 Ops.push_back(Builder.getInt1(
getTarget().isCLZForZeroUndef()));
7079 case NEON::BI__builtin_neon_vcvt_f32_v:
7080 case NEON::BI__builtin_neon_vcvtq_f32_v:
7081 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7084 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7085 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7086 case NEON::BI__builtin_neon_vcvt_f16_s16:
7087 case NEON::BI__builtin_neon_vcvt_f16_u16:
7088 case NEON::BI__builtin_neon_vcvtq_f16_s16:
7089 case NEON::BI__builtin_neon_vcvtq_f16_u16:
7090 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7093 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
7094 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
7095 case NEON::BI__builtin_neon_vcvt_n_f16_s16:
7096 case NEON::BI__builtin_neon_vcvt_n_f16_u16:
7097 case NEON::BI__builtin_neon_vcvtq_n_f16_s16:
7098 case NEON::BI__builtin_neon_vcvtq_n_f16_u16: {
7103 case NEON::BI__builtin_neon_vcvt_n_f32_v:
7104 case NEON::BI__builtin_neon_vcvt_n_f64_v:
7105 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
7106 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
7108 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
7112 case NEON::BI__builtin_neon_vcvt_n_s16_f16:
7113 case NEON::BI__builtin_neon_vcvt_n_s32_v:
7114 case NEON::BI__builtin_neon_vcvt_n_u16_f16:
7115 case NEON::BI__builtin_neon_vcvt_n_u32_v:
7116 case NEON::BI__builtin_neon_vcvt_n_s64_v:
7117 case NEON::BI__builtin_neon_vcvt_n_u64_v:
7118 case NEON::BI__builtin_neon_vcvtq_n_s16_f16:
7119 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
7120 case NEON::BI__builtin_neon_vcvtq_n_u16_f16:
7121 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
7122 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
7123 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
7128 case NEON::BI__builtin_neon_vcvt_s32_v:
7129 case NEON::BI__builtin_neon_vcvt_u32_v:
7130 case NEON::BI__builtin_neon_vcvt_s64_v:
7131 case NEON::BI__builtin_neon_vcvt_u64_v:
7132 case NEON::BI__builtin_neon_vcvt_s16_f16:
7133 case NEON::BI__builtin_neon_vcvt_u16_f16:
7134 case NEON::BI__builtin_neon_vcvtq_s32_v:
7135 case NEON::BI__builtin_neon_vcvtq_u32_v:
7136 case NEON::BI__builtin_neon_vcvtq_s64_v:
7137 case NEON::BI__builtin_neon_vcvtq_u64_v:
7138 case NEON::BI__builtin_neon_vcvtq_s16_f16:
7139 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
7141 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
7142 : Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
7144 case NEON::BI__builtin_neon_vcvta_s16_f16:
7145 case NEON::BI__builtin_neon_vcvta_s32_v:
7146 case NEON::BI__builtin_neon_vcvta_s64_v:
7147 case NEON::BI__builtin_neon_vcvta_u16_f16:
7148 case NEON::BI__builtin_neon_vcvta_u32_v:
7149 case NEON::BI__builtin_neon_vcvta_u64_v:
7150 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
7151 case NEON::BI__builtin_neon_vcvtaq_s32_v:
7152 case NEON::BI__builtin_neon_vcvtaq_s64_v:
7153 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
7154 case NEON::BI__builtin_neon_vcvtaq_u32_v:
7155 case NEON::BI__builtin_neon_vcvtaq_u64_v:
7156 case NEON::BI__builtin_neon_vcvtn_s16_f16:
7157 case NEON::BI__builtin_neon_vcvtn_s32_v:
7158 case NEON::BI__builtin_neon_vcvtn_s64_v:
7159 case NEON::BI__builtin_neon_vcvtn_u16_f16:
7160 case NEON::BI__builtin_neon_vcvtn_u32_v:
7161 case NEON::BI__builtin_neon_vcvtn_u64_v:
7162 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
7163 case NEON::BI__builtin_neon_vcvtnq_s32_v:
7164 case NEON::BI__builtin_neon_vcvtnq_s64_v:
7165 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
7166 case NEON::BI__builtin_neon_vcvtnq_u32_v:
7167 case NEON::BI__builtin_neon_vcvtnq_u64_v:
7168 case NEON::BI__builtin_neon_vcvtp_s16_f16:
7169 case NEON::BI__builtin_neon_vcvtp_s32_v:
7170 case NEON::BI__builtin_neon_vcvtp_s64_v:
7171 case NEON::BI__builtin_neon_vcvtp_u16_f16:
7172 case NEON::BI__builtin_neon_vcvtp_u32_v:
7173 case NEON::BI__builtin_neon_vcvtp_u64_v:
7174 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
7175 case NEON::BI__builtin_neon_vcvtpq_s32_v:
7176 case NEON::BI__builtin_neon_vcvtpq_s64_v:
7177 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
7178 case NEON::BI__builtin_neon_vcvtpq_u32_v:
7179 case NEON::BI__builtin_neon_vcvtpq_u64_v:
7180 case NEON::BI__builtin_neon_vcvtm_s16_f16:
7181 case NEON::BI__builtin_neon_vcvtm_s32_v:
7182 case NEON::BI__builtin_neon_vcvtm_s64_v:
7183 case NEON::BI__builtin_neon_vcvtm_u16_f16:
7184 case NEON::BI__builtin_neon_vcvtm_u32_v:
7185 case NEON::BI__builtin_neon_vcvtm_u64_v:
7186 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
7187 case NEON::BI__builtin_neon_vcvtmq_s32_v:
7188 case NEON::BI__builtin_neon_vcvtmq_s64_v:
7189 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
7190 case NEON::BI__builtin_neon_vcvtmq_u32_v:
7191 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
7195 case NEON::BI__builtin_neon_vcvtx_f32_v: {
7196 llvm::Type *Tys[2] = { VTy->getTruncatedElementVectorType(VTy), Ty};
7200 case NEON::BI__builtin_neon_vext_v:
7201 case NEON::BI__builtin_neon_vextq_v: {
7204 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7205 Indices.push_back(i+CV);
7207 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7208 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7209 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
7211 case NEON::BI__builtin_neon_vfma_v:
7212 case NEON::BI__builtin_neon_vfmaq_v: {
7213 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7214 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7215 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7219 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
7220 {Ops[1], Ops[2], Ops[0]});
7222 case NEON::BI__builtin_neon_vld1_v:
7223 case NEON::BI__builtin_neon_vld1q_v: {
7225 Ops.push_back(getAlignmentValue32(PtrOp0));
7228 case NEON::BI__builtin_neon_vld1_x2_v:
7229 case NEON::BI__builtin_neon_vld1q_x2_v:
7230 case NEON::BI__builtin_neon_vld1_x3_v:
7231 case NEON::BI__builtin_neon_vld1q_x3_v:
7232 case NEON::BI__builtin_neon_vld1_x4_v:
7233 case NEON::BI__builtin_neon_vld1q_x4_v: {
7234 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
7235 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
7236 llvm::Type *Tys[2] = { VTy, PTy };
7238 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld1xN");
7239 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
7240 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7241 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7243 case NEON::BI__builtin_neon_vld2_v:
7244 case NEON::BI__builtin_neon_vld2q_v:
7245 case NEON::BI__builtin_neon_vld3_v:
7246 case NEON::BI__builtin_neon_vld3q_v:
7247 case NEON::BI__builtin_neon_vld4_v:
7248 case NEON::BI__builtin_neon_vld4q_v:
7249 case NEON::BI__builtin_neon_vld2_dup_v:
7250 case NEON::BI__builtin_neon_vld2q_dup_v:
7251 case NEON::BI__builtin_neon_vld3_dup_v:
7252 case NEON::BI__builtin_neon_vld3q_dup_v:
7253 case NEON::BI__builtin_neon_vld4_dup_v:
7254 case NEON::BI__builtin_neon_vld4q_dup_v: {
7257 Value *Align = getAlignmentValue32(PtrOp1);
7258 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
7259 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
7260 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7261 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7263 case NEON::BI__builtin_neon_vld1_dup_v:
7264 case NEON::BI__builtin_neon_vld1q_dup_v: {
7265 Value *
V = PoisonValue::get(Ty);
7267 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
7268 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
7269 Ops[0] = Builder.CreateInsertElement(
V, Ld, CI);
7272 case NEON::BI__builtin_neon_vld2_lane_v:
7273 case NEON::BI__builtin_neon_vld2q_lane_v:
7274 case NEON::BI__builtin_neon_vld3_lane_v:
7275 case NEON::BI__builtin_neon_vld3q_lane_v:
7276 case NEON::BI__builtin_neon_vld4_lane_v:
7277 case NEON::BI__builtin_neon_vld4q_lane_v: {
7280 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
7281 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
7282 Ops.push_back(getAlignmentValue32(PtrOp1));
7283 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1), NameHint);
7284 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
7285 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7286 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
7288 case NEON::BI__builtin_neon_vmovl_v: {
7289 llvm::FixedVectorType *DTy =
7290 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7291 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
7293 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
7294 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
7296 case NEON::BI__builtin_neon_vmovn_v: {
7297 llvm::FixedVectorType *QTy =
7298 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7299 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
7300 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
7302 case NEON::BI__builtin_neon_vmull_v:
7308 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
7309 Int =
Type.isPoly() ? (
unsigned)Intrinsic::arm_neon_vmullp : Int;
7311 case NEON::BI__builtin_neon_vpadal_v:
7312 case NEON::BI__builtin_neon_vpadalq_v: {
7314 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7318 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7319 llvm::Type *Tys[2] = { Ty, NarrowTy };
7322 case NEON::BI__builtin_neon_vpaddl_v:
7323 case NEON::BI__builtin_neon_vpaddlq_v: {
7325 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
7326 llvm::Type *EltTy = llvm::IntegerType::get(
getLLVMContext(), EltBits / 2);
7328 llvm::FixedVectorType::get(EltTy, VTy->getNumElements() * 2);
7329 llvm::Type *Tys[2] = { Ty, NarrowTy };
7332 case NEON::BI__builtin_neon_vqdmlal_v:
7333 case NEON::BI__builtin_neon_vqdmlsl_v: {
7340 case NEON::BI__builtin_neon_vqdmulhq_lane_v:
7341 case NEON::BI__builtin_neon_vqdmulh_lane_v:
7342 case NEON::BI__builtin_neon_vqrdmulhq_lane_v:
7343 case NEON::BI__builtin_neon_vqrdmulh_lane_v: {
7345 if (BuiltinID == NEON::BI__builtin_neon_vqdmulhq_lane_v ||
7346 BuiltinID == NEON::BI__builtin_neon_vqrdmulhq_lane_v)
7347 RTy = llvm::FixedVectorType::get(RTy->getElementType(),
7348 RTy->getNumElements() * 2);
7349 llvm::Type *Tys[2] = {
7354 case NEON::BI__builtin_neon_vqdmulhq_laneq_v:
7355 case NEON::BI__builtin_neon_vqdmulh_laneq_v:
7356 case NEON::BI__builtin_neon_vqrdmulhq_laneq_v:
7357 case NEON::BI__builtin_neon_vqrdmulh_laneq_v: {
7358 llvm::Type *Tys[2] = {
7363 case NEON::BI__builtin_neon_vqshl_n_v:
7364 case NEON::BI__builtin_neon_vqshlq_n_v:
7367 case NEON::BI__builtin_neon_vqshlu_n_v:
7368 case NEON::BI__builtin_neon_vqshluq_n_v:
7371 case NEON::BI__builtin_neon_vrecpe_v:
7372 case NEON::BI__builtin_neon_vrecpeq_v:
7373 case NEON::BI__builtin_neon_vrsqrte_v:
7374 case NEON::BI__builtin_neon_vrsqrteq_v:
7375 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
7377 case NEON::BI__builtin_neon_vrndi_v:
7378 case NEON::BI__builtin_neon_vrndiq_v:
7379 Int = Builder.getIsFPConstrained()
7380 ? Intrinsic::experimental_constrained_nearbyint
7381 : Intrinsic::nearbyint;
7383 case NEON::BI__builtin_neon_vrshr_n_v:
7384 case NEON::BI__builtin_neon_vrshrq_n_v:
7387 case NEON::BI__builtin_neon_vsha512hq_u64:
7388 case NEON::BI__builtin_neon_vsha512h2q_u64:
7389 case NEON::BI__builtin_neon_vsha512su0q_u64:
7390 case NEON::BI__builtin_neon_vsha512su1q_u64: {
7394 case NEON::BI__builtin_neon_vshl_n_v:
7395 case NEON::BI__builtin_neon_vshlq_n_v:
7397 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
7399 case NEON::BI__builtin_neon_vshll_n_v: {
7400 llvm::FixedVectorType *SrcTy =
7401 llvm::FixedVectorType::getTruncatedElementVectorType(VTy);
7402 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7404 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
7406 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
7408 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
7410 case NEON::BI__builtin_neon_vshrn_n_v: {
7411 llvm::FixedVectorType *SrcTy =
7412 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7413 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7416 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
7418 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
7419 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
7421 case NEON::BI__builtin_neon_vshr_n_v:
7422 case NEON::BI__builtin_neon_vshrq_n_v:
7424 case NEON::BI__builtin_neon_vst1_v:
7425 case NEON::BI__builtin_neon_vst1q_v:
7426 case NEON::BI__builtin_neon_vst2_v:
7427 case NEON::BI__builtin_neon_vst2q_v:
7428 case NEON::BI__builtin_neon_vst3_v:
7429 case NEON::BI__builtin_neon_vst3q_v:
7430 case NEON::BI__builtin_neon_vst4_v:
7431 case NEON::BI__builtin_neon_vst4q_v:
7432 case NEON::BI__builtin_neon_vst2_lane_v:
7433 case NEON::BI__builtin_neon_vst2q_lane_v:
7434 case NEON::BI__builtin_neon_vst3_lane_v:
7435 case NEON::BI__builtin_neon_vst3q_lane_v:
7436 case NEON::BI__builtin_neon_vst4_lane_v:
7437 case NEON::BI__builtin_neon_vst4q_lane_v: {
7439 Ops.push_back(getAlignmentValue32(PtrOp0));
7442 case NEON::BI__builtin_neon_vsm3partw1q_u32:
7443 case NEON::BI__builtin_neon_vsm3partw2q_u32:
7444 case NEON::BI__builtin_neon_vsm3ss1q_u32:
7445 case NEON::BI__builtin_neon_vsm4ekeyq_u32:
7446 case NEON::BI__builtin_neon_vsm4eq_u32: {
7450 case NEON::BI__builtin_neon_vsm3tt1aq_u32:
7451 case NEON::BI__builtin_neon_vsm3tt1bq_u32:
7452 case NEON::BI__builtin_neon_vsm3tt2aq_u32:
7453 case NEON::BI__builtin_neon_vsm3tt2bq_u32: {
7455 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
7458 case NEON::BI__builtin_neon_vst1_x2_v:
7459 case NEON::BI__builtin_neon_vst1q_x2_v:
7460 case NEON::BI__builtin_neon_vst1_x3_v:
7461 case NEON::BI__builtin_neon_vst1q_x3_v:
7462 case NEON::BI__builtin_neon_vst1_x4_v:
7463 case NEON::BI__builtin_neon_vst1q_x4_v: {
7464 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getElementType());
7467 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be ||
7468 Arch == llvm::Triple::aarch64_32) {
7469 llvm::Type *Tys[2] = { VTy, PTy };
7470 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
7473 llvm::Type *Tys[2] = { PTy, VTy };
7476 case NEON::BI__builtin_neon_vsubhn_v: {
7477 llvm::FixedVectorType *SrcTy =
7478 llvm::FixedVectorType::getExtendedElementVectorType(VTy);
7481 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
7482 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
7483 Ops[0] = Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
7486 Constant *ShiftAmt =
7487 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
7488 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
7491 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
7493 case NEON::BI__builtin_neon_vtrn_v:
7494 case NEON::BI__builtin_neon_vtrnq_v: {
7495 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7496 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7497 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7498 Value *SV =
nullptr;
7500 for (
unsigned vi = 0; vi != 2; ++vi) {
7502 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7503 Indices.push_back(i+vi);
7504 Indices.push_back(i+e+vi);
7506 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7507 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
7508 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7512 case NEON::BI__builtin_neon_vtst_v:
7513 case NEON::BI__builtin_neon_vtstq_v: {
7514 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7515 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7516 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7517 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
7518 ConstantAggregateZero::get(Ty));
7519 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
7521 case NEON::BI__builtin_neon_vuzp_v:
7522 case NEON::BI__builtin_neon_vuzpq_v: {
7523 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7524 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7525 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7526 Value *SV =
nullptr;
7528 for (
unsigned vi = 0; vi != 2; ++vi) {
7530 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
7531 Indices.push_back(2*i+vi);
7533 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7534 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
7535 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7539 case NEON::BI__builtin_neon_vxarq_u64: {
7541 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
7544 case NEON::BI__builtin_neon_vzip_v:
7545 case NEON::BI__builtin_neon_vzipq_v: {
7546 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
7547 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7548 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7549 Value *SV =
nullptr;
7551 for (
unsigned vi = 0; vi != 2; ++vi) {
7553 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
7554 Indices.push_back((i + vi*e) >> 1);
7555 Indices.push_back(((i + vi*e) >> 1)+e);
7557 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
7558 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
7559 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
7563 case NEON::BI__builtin_neon_vdot_s32:
7564 case NEON::BI__builtin_neon_vdot_u32:
7565 case NEON::BI__builtin_neon_vdotq_s32:
7566 case NEON::BI__builtin_neon_vdotq_u32: {
7568 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7569 llvm::Type *Tys[2] = { Ty, InputTy };
7572 case NEON::BI__builtin_neon_vfmlal_low_f16:
7573 case NEON::BI__builtin_neon_vfmlalq_low_f16: {
7575 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7576 llvm::Type *Tys[2] = { Ty, InputTy };
7579 case NEON::BI__builtin_neon_vfmlsl_low_f16:
7580 case NEON::BI__builtin_neon_vfmlslq_low_f16: {
7582 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7583 llvm::Type *Tys[2] = { Ty, InputTy };
7586 case NEON::BI__builtin_neon_vfmlal_high_f16:
7587 case NEON::BI__builtin_neon_vfmlalq_high_f16: {
7589 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7590 llvm::Type *Tys[2] = { Ty, InputTy };
7593 case NEON::BI__builtin_neon_vfmlsl_high_f16:
7594 case NEON::BI__builtin_neon_vfmlslq_high_f16: {
7596 llvm::FixedVectorType::get(
HalfTy, Ty->getPrimitiveSizeInBits() / 16);
7597 llvm::Type *Tys[2] = { Ty, InputTy };
7600 case NEON::BI__builtin_neon_vmmlaq_s32:
7601 case NEON::BI__builtin_neon_vmmlaq_u32: {
7603 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7604 llvm::Type *Tys[2] = { Ty, InputTy };
7607 case NEON::BI__builtin_neon_vusmmlaq_s32: {
7609 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7610 llvm::Type *Tys[2] = { Ty, InputTy };
7613 case NEON::BI__builtin_neon_vusdot_s32:
7614 case NEON::BI__builtin_neon_vusdotq_s32: {
7616 llvm::FixedVectorType::get(
Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
7617 llvm::Type *Tys[2] = { Ty, InputTy };
7620 case NEON::BI__builtin_neon_vbfdot_f32:
7621 case NEON::BI__builtin_neon_vbfdotq_f32: {
7622 llvm::Type *InputTy =
7623 llvm::FixedVectorType::get(
BFloatTy, Ty->getPrimitiveSizeInBits() / 16);
7624 llvm::Type *Tys[2] = { Ty, InputTy };
7627 case NEON::BI__builtin_neon___a32_vcvt_bf16_f32: {
7628 llvm::Type *Tys[1] = { Ty };
7635 assert(Int &&
"Expected valid intrinsic number");
7644 return Builder.CreateBitCast(
Result, ResultType, NameHint);
7648 Value *Op, llvm::Type *Ty,
const CmpInst::Predicate Fp,
7649 const CmpInst::Predicate Ip,
const Twine &Name) {
7650 llvm::Type *OTy = Op->
getType();
7656 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
7657 OTy = BI->getOperand(0)->getType();
7659 Op = Builder.CreateBitCast(Op, OTy);
7660 if (OTy->getScalarType()->isFloatingPointTy()) {
7661 if (Fp == CmpInst::FCMP_OEQ)
7662 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
7664 Op = Builder.CreateFCmpS(Fp, Op, Constant::getNullValue(OTy));
7666 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
7668 return Builder.CreateSExt(Op, Ty, Name);
7673 llvm::Type *ResTy,
unsigned IntID,
7677 TblOps.push_back(ExtOp);
7682 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
7683 Indices.push_back(2*i);
7684 Indices.push_back(2*i+1);
7687 int PairPos = 0, End = Ops.size() - 1;
7688 while (PairPos < End) {
7689 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
7690 Ops[PairPos+1], Indices,
7697 if (PairPos == End) {
7698 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
7699 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
7700 ZeroTbl, Indices, Name));
7704 TblOps.push_back(IndexOp);
7710Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
7712 switch (BuiltinID) {
7715 case clang::ARM::BI__builtin_arm_nop:
7718 case clang::ARM::BI__builtin_arm_yield:
7719 case clang::ARM::BI__yield:
7722 case clang::ARM::BI__builtin_arm_wfe:
7723 case clang::ARM::BI__wfe:
7726 case clang::ARM::BI__builtin_arm_wfi:
7727 case clang::ARM::BI__wfi:
7730 case clang::ARM::BI__builtin_arm_sev:
7731 case clang::ARM::BI__sev:
7734 case clang::ARM::BI__builtin_arm_sevl:
7735 case clang::ARM::BI__sevl:
7755 llvm::Type *RegisterType,
7756 llvm::Type *ValueType,
7758 StringRef SysReg =
"") {
7760 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
7761 RegisterType->isIntegerTy(128)) &&
7762 "Unsupported size for register.");
7768 if (SysReg.empty()) {
7773 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
7774 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
7775 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
7777 llvm::Type *Types[] = { RegisterType };
7779 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
7780 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
7781 &&
"Can't fit 64-bit value in 32-bit register");
7783 if (AccessKind !=
Write) {
7786 AccessKind ==
VolatileRead ? llvm::Intrinsic::read_volatile_register
7787 : llvm::Intrinsic::read_register,
7789 llvm::Value *Call = Builder.CreateCall(F, Metadata);
7793 return Builder.CreateTrunc(Call, ValueType);
7795 if (ValueType->isPointerTy())
7797 return Builder.CreateIntToPtr(Call, ValueType);
7802 llvm::Function *F = CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
7806 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
7807 return Builder.CreateCall(F, { Metadata, ArgValue });
7810 if (ValueType->isPointerTy()) {
7812 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
7813 return Builder.CreateCall(F, { Metadata, ArgValue });
7816 return Builder.CreateCall(F, { Metadata, ArgValue });
7822 switch (BuiltinID) {
7824 case NEON::BI__builtin_neon_vget_lane_i8:
7825 case NEON::BI__builtin_neon_vget_lane_i16:
7826 case NEON::BI__builtin_neon_vget_lane_bf16:
7827 case NEON::BI__builtin_neon_vget_lane_i32:
7828 case NEON::BI__builtin_neon_vget_lane_i64:
7829 case NEON::BI__builtin_neon_vget_lane_f32:
7830 case NEON::BI__builtin_neon_vgetq_lane_i8:
7831 case NEON::BI__builtin_neon_vgetq_lane_i16:
7832 case NEON::BI__builtin_neon_vgetq_lane_bf16:
7833 case NEON::BI__builtin_neon_vgetq_lane_i32:
7834 case NEON::BI__builtin_neon_vgetq_lane_i64:
7835 case NEON::BI__builtin_neon_vgetq_lane_f32:
7836 case NEON::BI__builtin_neon_vduph_lane_bf16:
7837 case NEON::BI__builtin_neon_vduph_laneq_bf16:
7838 case NEON::BI__builtin_neon_vset_lane_i8:
7839 case NEON::BI__builtin_neon_vset_lane_i16:
7840 case NEON::BI__builtin_neon_vset_lane_bf16:
7841 case NEON::BI__builtin_neon_vset_lane_i32:
7842 case NEON::BI__builtin_neon_vset_lane_i64:
7843 case NEON::BI__builtin_neon_vset_lane_f32:
7844 case NEON::BI__builtin_neon_vsetq_lane_i8:
7845 case NEON::BI__builtin_neon_vsetq_lane_i16:
7846 case NEON::BI__builtin_neon_vsetq_lane_bf16:
7847 case NEON::BI__builtin_neon_vsetq_lane_i32:
7848 case NEON::BI__builtin_neon_vsetq_lane_i64:
7849 case NEON::BI__builtin_neon_vsetq_lane_f32:
7850 case NEON::BI__builtin_neon_vsha1h_u32:
7851 case NEON::BI__builtin_neon_vsha1cq_u32:
7852 case NEON::BI__builtin_neon_vsha1pq_u32:
7853 case NEON::BI__builtin_neon_vsha1mq_u32:
7854 case NEON::BI__builtin_neon_vcvth_bf16_f32:
7855 case clang::ARM::BI_MoveToCoprocessor:
7856 case clang::ARM::BI_MoveToCoprocessor2:
7865 llvm::Triple::ArchType Arch) {
7866 if (
auto Hint = GetValueForARMHint(BuiltinID))
7869 if (BuiltinID == clang::ARM::BI__emit) {
7871 llvm::FunctionType *FTy =
7872 llvm::FunctionType::get(
VoidTy,
false);
7876 llvm_unreachable(
"Sema will ensure that the parameter is constant");
7879 uint64_t ZExtValue =
Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
7881 llvm::InlineAsm *Emit =
7882 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
7884 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
7887 return Builder.CreateCall(Emit);
7890 if (BuiltinID == clang::ARM::BI__builtin_arm_dbg) {
7892 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_dbg), Option);
7895 if (BuiltinID == clang::ARM::BI__builtin_arm_prefetch) {
7904 return Builder.CreateCall(F, {
Address, RW, Locality, IsData});
7907 if (BuiltinID == clang::ARM::BI__builtin_arm_rbit) {
7909 return Builder.CreateCall(
7910 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
7913 if (BuiltinID == clang::ARM::BI__builtin_arm_clz ||
7914 BuiltinID == clang::ARM::BI__builtin_arm_clz64) {
7917 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(
false)});
7918 if (BuiltinID == clang::ARM::BI__builtin_arm_clz64)
7919 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
7924 if (BuiltinID == clang::ARM::BI__builtin_arm_cls) {
7926 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_cls), Arg,
"cls");
7928 if (BuiltinID == clang::ARM::BI__builtin_arm_cls64) {
7930 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::arm_cls64), Arg,
7934 if (BuiltinID == clang::ARM::BI__clear_cache) {
7935 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
7938 for (
unsigned i = 0; i < 2; i++)
7942 StringRef Name = FD->
getName();
7946 if (BuiltinID == clang::ARM::BI__builtin_arm_mcrr ||
7947 BuiltinID == clang::ARM::BI__builtin_arm_mcrr2) {
7950 switch (BuiltinID) {
7951 default: llvm_unreachable(
"unexpected builtin");
7952 case clang::ARM::BI__builtin_arm_mcrr:
7955 case clang::ARM::BI__builtin_arm_mcrr2:
7973 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2,
Int32Ty);
7974 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
7975 Rt2 = Builder.CreateTruncOrBitCast(Rt2,
Int32Ty);
7977 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
7980 if (BuiltinID == clang::ARM::BI__builtin_arm_mrrc ||
7981 BuiltinID == clang::ARM::BI__builtin_arm_mrrc2) {
7984 switch (BuiltinID) {
7985 default: llvm_unreachable(
"unexpected builtin");
7986 case clang::ARM::BI__builtin_arm_mrrc:
7989 case clang::ARM::BI__builtin_arm_mrrc2:
7997 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
8002 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
8003 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
8004 Rt = Builder.CreateZExt(Rt,
Int64Ty);
8005 Rt1 = Builder.CreateZExt(Rt1,
Int64Ty);
8007 Value *ShiftCast = llvm::ConstantInt::get(
Int64Ty, 32);
8008 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
8009 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
8014 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrexd ||
8015 ((BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8016 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) &&
8018 BuiltinID == clang::ARM::BI__ldrexd) {
8021 switch (BuiltinID) {
8022 default: llvm_unreachable(
"unexpected builtin");
8023 case clang::ARM::BI__builtin_arm_ldaex:
8026 case clang::ARM::BI__builtin_arm_ldrexd:
8027 case clang::ARM::BI__builtin_arm_ldrex:
8028 case clang::ARM::BI__ldrexd:
8034 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr,
Int8PtrTy),
8037 Value *Val0 = Builder.CreateExtractValue(Val, 1);
8038 Value *Val1 = Builder.CreateExtractValue(Val, 0);
8039 Val0 = Builder.CreateZExt(Val0,
Int64Ty);
8040 Val1 = Builder.CreateZExt(Val1,
Int64Ty);
8043 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
8044 Val = Builder.CreateOr(Val, Val1);
8048 if (BuiltinID == clang::ARM::BI__builtin_arm_ldrex ||
8049 BuiltinID == clang::ARM::BI__builtin_arm_ldaex) {
8056 llvm::Type *PtrTy = llvm::PointerType::getUnqual(
getLLVMContext());
8059 BuiltinID == clang::ARM::BI__builtin_arm_ldaex ? Intrinsic::arm_ldaex
8060 : Intrinsic::arm_ldrex,
8062 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldrex");
8066 if (RealResTy->isPointerTy())
8067 return Builder.CreateIntToPtr(Val, RealResTy);
8069 llvm::Type *IntResTy = llvm::IntegerType::get(
8071 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
8076 if (BuiltinID == clang::ARM::BI__builtin_arm_strexd ||
8077 ((BuiltinID == clang::ARM::BI__builtin_arm_stlex ||
8078 BuiltinID == clang::ARM::BI__builtin_arm_strex) &&
8081 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlexd
8082 : Intrinsic::arm_strexd);
8087 Builder.CreateStore(Val, Tmp);
8090 Val = Builder.CreateLoad(LdPtr);
8092 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
8093 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
8095 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
8098 if (BuiltinID == clang::ARM::BI__builtin_arm_strex ||
8099 BuiltinID == clang::ARM::BI__builtin_arm_stlex) {
8104 llvm::Type *StoreTy =
8107 if (StoreVal->
getType()->isPointerTy())
8108 StoreVal = Builder.CreatePtrToInt(StoreVal,
Int32Ty);
8110 llvm::Type *
IntTy = llvm::IntegerType::get(
8113 StoreVal = Builder.CreateBitCast(StoreVal,
IntTy);
8114 StoreVal = Builder.CreateZExtOrBitCast(StoreVal,
Int32Ty);
8118 BuiltinID == clang::ARM::BI__builtin_arm_stlex ? Intrinsic::arm_stlex
8119 : Intrinsic::arm_strex,
8122 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
8124 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
8128 if (BuiltinID == clang::ARM::BI__builtin_arm_clrex) {
8130 return Builder.CreateCall(F);
8134 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
8135 switch (BuiltinID) {
8136 case clang::ARM::BI__builtin_arm_crc32b:
8137 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
8138 case clang::ARM::BI__builtin_arm_crc32cb:
8139 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
8140 case clang::ARM::BI__builtin_arm_crc32h:
8141 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
8142 case clang::ARM::BI__builtin_arm_crc32ch:
8143 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
8144 case clang::ARM::BI__builtin_arm_crc32w:
8145 case clang::ARM::BI__builtin_arm_crc32d:
8146 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
8147 case clang::ARM::BI__builtin_arm_crc32cw:
8148 case clang::ARM::BI__builtin_arm_crc32cd:
8149 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
8152 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
8158 if (BuiltinID == clang::ARM::BI__builtin_arm_crc32d ||
8159 BuiltinID == clang::ARM::BI__builtin_arm_crc32cd) {
8161 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1,
Int32Ty);
8162 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
8163 Arg1b = Builder.CreateTruncOrBitCast(Arg1b,
Int32Ty);
8166 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
8167 return Builder.CreateCall(F, {Res, Arg1b});
8169 Arg1 = Builder.CreateZExtOrBitCast(Arg1,
Int32Ty);
8172 return Builder.CreateCall(F, {Arg0, Arg1});
8176 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8177 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8178 BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8179 BuiltinID == clang::ARM::BI__builtin_arm_wsr ||
8180 BuiltinID == clang::ARM::BI__builtin_arm_wsr64 ||
8181 BuiltinID == clang::ARM::BI__builtin_arm_wsrp) {
8184 if (BuiltinID == clang::ARM::BI__builtin_arm_rsr ||
8185 BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8186 BuiltinID == clang::ARM::BI__builtin_arm_rsrp)
8189 bool IsPointerBuiltin = BuiltinID == clang::ARM::BI__builtin_arm_rsrp ||
8190 BuiltinID == clang::ARM::BI__builtin_arm_wsrp;
8192 bool Is64Bit = BuiltinID == clang::ARM::BI__builtin_arm_rsr64 ||
8193 BuiltinID == clang::ARM::BI__builtin_arm_wsr64;
8195 llvm::Type *ValueType;
8196 llvm::Type *RegisterType;
8197 if (IsPointerBuiltin) {
8200 }
else if (Is64Bit) {
8201 ValueType = RegisterType =
Int64Ty;
8203 ValueType = RegisterType =
Int32Ty;
8210 if (BuiltinID == ARM::BI__builtin_sponentry) {
8212 return Builder.CreateCall(F);
8229 return P.first == BuiltinID;
8232 BuiltinID = It->second;
8236 unsigned ICEArguments = 0;
8241 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
8242 return Builder.getInt32(addr.getAlignment().getQuantity());
8249 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
8250 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
8252 switch (BuiltinID) {
8253 case NEON::BI__builtin_neon_vld1_v:
8254 case NEON::BI__builtin_neon_vld1q_v:
8255 case NEON::BI__builtin_neon_vld1q_lane_v:
8256 case NEON::BI__builtin_neon_vld1_lane_v:
8257 case NEON::BI__builtin_neon_vld1_dup_v:
8258 case NEON::BI__builtin_neon_vld1q_dup_v:
8259 case NEON::BI__builtin_neon_vst1_v:
8260 case NEON::BI__builtin_neon_vst1q_v:
8261 case NEON::BI__builtin_neon_vst1q_lane_v:
8262 case NEON::BI__builtin_neon_vst1_lane_v:
8263 case NEON::BI__builtin_neon_vst2_v:
8264 case NEON::BI__builtin_neon_vst2q_v:
8265 case NEON::BI__builtin_neon_vst2_lane_v:
8266 case NEON::BI__builtin_neon_vst2q_lane_v:
8267 case NEON::BI__builtin_neon_vst3_v:
8268 case NEON::BI__builtin_neon_vst3q_v:
8269 case NEON::BI__builtin_neon_vst3_lane_v:
8270 case NEON::BI__builtin_neon_vst3q_lane_v:
8271 case NEON::BI__builtin_neon_vst4_v:
8272 case NEON::BI__builtin_neon_vst4q_v:
8273 case NEON::BI__builtin_neon_vst4_lane_v:
8274 case NEON::BI__builtin_neon_vst4q_lane_v:
8283 switch (BuiltinID) {
8284 case NEON::BI__builtin_neon_vld2_v:
8285 case NEON::BI__builtin_neon_vld2q_v:
8286 case NEON::BI__builtin_neon_vld3_v:
8287 case NEON::BI__builtin_neon_vld3q_v:
8288 case NEON::BI__builtin_neon_vld4_v:
8289 case NEON::BI__builtin_neon_vld4q_v:
8290 case NEON::BI__builtin_neon_vld2_lane_v:
8291 case NEON::BI__builtin_neon_vld2q_lane_v:
8292 case NEON::BI__builtin_neon_vld3_lane_v:
8293 case NEON::BI__builtin_neon_vld3q_lane_v:
8294 case NEON::BI__builtin_neon_vld4_lane_v:
8295 case NEON::BI__builtin_neon_vld4q_lane_v:
8296 case NEON::BI__builtin_neon_vld2_dup_v:
8297 case NEON::BI__builtin_neon_vld2q_dup_v:
8298 case NEON::BI__builtin_neon_vld3_dup_v:
8299 case NEON::BI__builtin_neon_vld3q_dup_v:
8300 case NEON::BI__builtin_neon_vld4_dup_v:
8301 case NEON::BI__builtin_neon_vld4q_dup_v:
8310 if ((ICEArguments & (1 << i)) == 0) {
8315 Ops.push_back(llvm::ConstantInt::get(
8321 switch (BuiltinID) {
8324 case NEON::BI__builtin_neon_vget_lane_i8:
8325 case NEON::BI__builtin_neon_vget_lane_i16:
8326 case NEON::BI__builtin_neon_vget_lane_i32:
8327 case NEON::BI__builtin_neon_vget_lane_i64:
8328 case NEON::BI__builtin_neon_vget_lane_bf16:
8329 case NEON::BI__builtin_neon_vget_lane_f32:
8330 case NEON::BI__builtin_neon_vgetq_lane_i8:
8331 case NEON::BI__builtin_neon_vgetq_lane_i16:
8332 case NEON::BI__builtin_neon_vgetq_lane_i32:
8333 case NEON::BI__builtin_neon_vgetq_lane_i64:
8334 case NEON::BI__builtin_neon_vgetq_lane_bf16:
8335 case NEON::BI__builtin_neon_vgetq_lane_f32:
8336 case NEON::BI__builtin_neon_vduph_lane_bf16:
8337 case NEON::BI__builtin_neon_vduph_laneq_bf16:
8338 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
8340 case NEON::BI__builtin_neon_vrndns_f32: {
8342 llvm::Type *Tys[] = {Arg->
getType()};
8344 return Builder.CreateCall(F, {Arg},
"vrndn"); }
8346 case NEON::BI__builtin_neon_vset_lane_i8:
8347 case NEON::BI__builtin_neon_vset_lane_i16:
8348 case NEON::BI__builtin_neon_vset_lane_i32:
8349 case NEON::BI__builtin_neon_vset_lane_i64:
8350 case NEON::BI__builtin_neon_vset_lane_bf16:
8351 case NEON::BI__builtin_neon_vset_lane_f32:
8352 case NEON::BI__builtin_neon_vsetq_lane_i8:
8353 case NEON::BI__builtin_neon_vsetq_lane_i16:
8354 case NEON::BI__builtin_neon_vsetq_lane_i32:
8355 case NEON::BI__builtin_neon_vsetq_lane_i64:
8356 case NEON::BI__builtin_neon_vsetq_lane_bf16:
8357 case NEON::BI__builtin_neon_vsetq_lane_f32:
8358 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
8360 case NEON::BI__builtin_neon_vsha1h_u32:
8363 case NEON::BI__builtin_neon_vsha1cq_u32:
8366 case NEON::BI__builtin_neon_vsha1pq_u32:
8369 case NEON::BI__builtin_neon_vsha1mq_u32:
8373 case NEON::BI__builtin_neon_vcvth_bf16_f32: {
8380 case clang::ARM::BI_MoveToCoprocessor:
8381 case clang::ARM::BI_MoveToCoprocessor2: {
8382 Function *F =
CGM.
getIntrinsic(BuiltinID == clang::ARM::BI_MoveToCoprocessor
8383 ? Intrinsic::arm_mcr
8384 : Intrinsic::arm_mcr2);
8385 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
8386 Ops[3], Ops[4], Ops[5]});
8391 assert(HasExtraArg);
8393 std::optional<llvm::APSInt>
Result =
8398 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f ||
8399 BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_d) {
8402 if (BuiltinID == clang::ARM::BI__builtin_arm_vcvtr_f)
8408 bool usgn =
Result->getZExtValue() == 1;
8409 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
8413 return Builder.CreateCall(F, Ops,
"vcvtr");
8418 bool usgn =
Type.isUnsigned();
8419 bool rightShift =
false;
8421 llvm::FixedVectorType *VTy =
8424 llvm::Type *Ty = VTy;
8435 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
8436 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
8439 switch (BuiltinID) {
8440 default:
return nullptr;
8441 case NEON::BI__builtin_neon_vld1q_lane_v:
8444 if (VTy->getElementType()->isIntegerTy(64)) {
8446 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8448 Value *SV = llvm::ConstantVector::get(ConstantInt::get(
Int32Ty, 1-Lane));
8449 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
8451 Ty = llvm::FixedVectorType::get(VTy->getElementType(), 1);
8454 Value *Align = getAlignmentValue32(PtrOp0);
8455 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
8457 int Indices[] = {1 - Lane, Lane};
8458 return Builder.CreateShuffleVector(Ops[1], Ld, Indices,
"vld1q_lane");
8461 case NEON::BI__builtin_neon_vld1_lane_v: {
8462 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8464 Value *Ld = Builder.CreateLoad(PtrOp0);
8465 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
8467 case NEON::BI__builtin_neon_vqrshrn_n_v:
8469 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
8472 case NEON::BI__builtin_neon_vqrshrun_n_v:
8474 Ops,
"vqrshrun_n", 1,
true);
8475 case NEON::BI__builtin_neon_vqshrn_n_v:
8476 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
8479 case NEON::BI__builtin_neon_vqshrun_n_v:
8481 Ops,
"vqshrun_n", 1,
true);
8482 case NEON::BI__builtin_neon_vrecpe_v:
8483 case NEON::BI__builtin_neon_vrecpeq_v:
8486 case NEON::BI__builtin_neon_vrshrn_n_v:
8488 Ops,
"vrshrn_n", 1,
true);
8489 case NEON::BI__builtin_neon_vrsra_n_v:
8490 case NEON::BI__builtin_neon_vrsraq_n_v:
8491 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8492 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8494 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
8495 Ops[1] = Builder.CreateCall(
CGM.
getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
8496 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
8497 case NEON::BI__builtin_neon_vsri_n_v:
8498 case NEON::BI__builtin_neon_vsriq_n_v:
8501 case NEON::BI__builtin_neon_vsli_n_v:
8502 case NEON::BI__builtin_neon_vsliq_n_v:
8506 case NEON::BI__builtin_neon_vsra_n_v:
8507 case NEON::BI__builtin_neon_vsraq_n_v:
8508 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8510 return Builder.CreateAdd(Ops[0], Ops[1]);
8511 case NEON::BI__builtin_neon_vst1q_lane_v:
8514 if (VTy->getElementType()->isIntegerTy(64)) {
8515 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8516 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
8517 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
8518 Ops[2] = getAlignmentValue32(PtrOp0);
8519 llvm::Type *Tys[] = {
Int8PtrTy, Ops[1]->getType()};
8524 case NEON::BI__builtin_neon_vst1_lane_v: {
8525 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8526 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
8527 return Builder.CreateStore(Ops[1],
8530 case NEON::BI__builtin_neon_vtbl1_v:
8533 case NEON::BI__builtin_neon_vtbl2_v:
8536 case NEON::BI__builtin_neon_vtbl3_v:
8539 case NEON::BI__builtin_neon_vtbl4_v:
8542 case NEON::BI__builtin_neon_vtbx1_v:
8545 case NEON::BI__builtin_neon_vtbx2_v:
8548 case NEON::BI__builtin_neon_vtbx3_v:
8551 case NEON::BI__builtin_neon_vtbx4_v:
8557template<
typename Integer>
8566 return Unsigned ? Builder.CreateZExt(
V, T) : Builder.CreateSExt(
V, T);
8578 ->getPrimitiveSizeInBits();
8579 if (Shift == LaneBits) {
8584 return llvm::Constant::getNullValue(
V->getType());
8588 return Unsigned ? Builder.CreateLShr(
V, Shift) : Builder.CreateAShr(
V, Shift);
8595 unsigned Elements = 128 /
V->getType()->getPrimitiveSizeInBits();
8596 return Builder.CreateVectorSplat(Elements,
V);
8602 llvm::Type *DestType) {
8615 V->getType()->getScalarSizeInBits() != DestType->getScalarSizeInBits()) {
8616 return Builder.CreateCall(
8618 {DestType, V->getType()}),
8621 return Builder.CreateBitCast(
V, DestType);
8629 unsigned InputElements =
8631 for (
unsigned i = 0; i < InputElements; i += 2)
8632 Indices.push_back(i + Odd);
8633 return Builder.CreateShuffleVector(
V, Indices);
8639 assert(V0->getType() == V1->getType() &&
"Can't zip different vector types");
8641 unsigned InputElements =
8643 for (
unsigned i = 0; i < InputElements; i++) {
8644 Indices.push_back(i);
8645 Indices.push_back(i + InputElements);
8647 return Builder.CreateShuffleVector(V0, V1, Indices);
8650template<
unsigned HighBit,
unsigned OtherBits>
8655 unsigned LaneBits = T->getPrimitiveSizeInBits();
8656 uint32_t
Value = HighBit << (LaneBits - 1);
8658 Value |= (1UL << (LaneBits - 1)) - 1;
8659 llvm::Value *Lane = llvm::ConstantInt::get(T,
Value);
8665 unsigned ReverseWidth) {
8669 unsigned LaneSize =
V->getType()->getScalarSizeInBits();
8670 unsigned Elements = 128 / LaneSize;
8671 unsigned Mask = ReverseWidth / LaneSize - 1;
8672 for (
unsigned i = 0; i < Elements; i++)
8673 Indices.push_back(i ^ Mask);
8674 return Builder.CreateShuffleVector(
V, Indices);
8680 llvm::Triple::ArchType Arch) {
8681 enum class CustomCodeGen { VLD24, VST24 } CustomCodeGenType;
8682 Intrinsic::ID IRIntr;
8683 unsigned NumVectors;
8686 switch (BuiltinID) {
8687 #include "clang/Basic/arm_mve_builtin_cg.inc"
8698 switch (CustomCodeGenType) {
8700 case CustomCodeGen::VLD24: {
8704 auto MvecCType = E->
getType();
8706 assert(MvecLType->isStructTy() &&
8707 "Return type for vld[24]q should be a struct");
8708 assert(MvecLType->getStructNumElements() == 1 &&
8709 "Return-type struct for vld[24]q should have one element");
8710 auto MvecLTypeInner = MvecLType->getStructElementType(0);
8711 assert(MvecLTypeInner->isArrayTy() &&
8712 "Return-type struct for vld[24]q should contain an array");
8713 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
8714 "Array member of return-type struct vld[24]q has wrong length");
8715 auto VecLType = MvecLTypeInner->getArrayElementType();
8717 Tys.push_back(VecLType);
8719 auto Addr = E->
getArg(0);
8724 Value *LoadResult = Builder.CreateCall(F, Ops);
8725 Value *MvecOut = PoisonValue::get(MvecLType);
8726 for (
unsigned i = 0; i < NumVectors; ++i) {
8727 Value *Vec = Builder.CreateExtractValue(LoadResult, i);
8728 MvecOut = Builder.CreateInsertValue(MvecOut, Vec, {0, i});
8734 return Builder.CreateStore(MvecOut,
ReturnValue.getValue());
8737 case CustomCodeGen::VST24: {
8741 auto Addr = E->
getArg(0);
8747 assert(MvecLType->isStructTy() &&
"Data type for vst2q should be a struct");
8748 assert(MvecLType->getStructNumElements() == 1 &&
8749 "Data-type struct for vst2q should have one element");
8750 auto MvecLTypeInner = MvecLType->getStructElementType(0);
8751 assert(MvecLTypeInner->isArrayTy() &&
8752 "Data-type struct for vst2q should contain an array");
8753 assert(MvecLTypeInner->getArrayNumElements() == NumVectors &&
8754 "Array member of return-type struct vld[24]q has wrong length");
8755 auto VecLType = MvecLTypeInner->getArrayElementType();
8757 Tys.push_back(VecLType);
8761 auto Mvec = Builder.CreateLoad(MvecSlot.
getAddress());
8762 for (
unsigned i = 0; i < NumVectors; i++)
8763 Ops.push_back(Builder.CreateExtractValue(Mvec, {0, i}));
8766 Value *ToReturn =
nullptr;
8767 for (
unsigned i = 0; i < NumVectors; i++) {
8768 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, i));
8769 ToReturn = Builder.CreateCall(F, Ops);
8775 llvm_unreachable(
"unknown custom codegen type.");
8781 llvm::Triple::ArchType Arch) {
8782 switch (BuiltinID) {
8785#include "clang/Basic/arm_cde_builtin_cg.inc"
8792 llvm::Triple::ArchType Arch) {
8793 unsigned int Int = 0;
8794 const char *
s =
nullptr;
8796 switch (BuiltinID) {
8799 case NEON::BI__builtin_neon_vtbl1_v:
8800 case NEON::BI__builtin_neon_vqtbl1_v:
8801 case NEON::BI__builtin_neon_vqtbl1q_v:
8802 case NEON::BI__builtin_neon_vtbl2_v:
8803 case NEON::BI__builtin_neon_vqtbl2_v:
8804 case NEON::BI__builtin_neon_vqtbl2q_v:
8805 case NEON::BI__builtin_neon_vtbl3_v:
8806 case NEON::BI__builtin_neon_vqtbl3_v:
8807 case NEON::BI__builtin_neon_vqtbl3q_v:
8808 case NEON::BI__builtin_neon_vtbl4_v:
8809 case NEON::BI__builtin_neon_vqtbl4_v:
8810 case NEON::BI__builtin_neon_vqtbl4q_v:
8812 case NEON::BI__builtin_neon_vtbx1_v:
8813 case NEON::BI__builtin_neon_vqtbx1_v:
8814 case NEON::BI__builtin_neon_vqtbx1q_v:
8815 case NEON::BI__builtin_neon_vtbx2_v:
8816 case NEON::BI__builtin_neon_vqtbx2_v:
8817 case NEON::BI__builtin_neon_vqtbx2q_v:
8818 case NEON::BI__builtin_neon_vtbx3_v:
8819 case NEON::BI__builtin_neon_vqtbx3_v:
8820 case NEON::BI__builtin_neon_vqtbx3q_v:
8821 case NEON::BI__builtin_neon_vtbx4_v:
8822 case NEON::BI__builtin_neon_vqtbx4_v:
8823 case NEON::BI__builtin_neon_vqtbx4q_v:
8831 std::optional<llvm::APSInt>
Result =
8846 switch (BuiltinID) {
8847 case NEON::BI__builtin_neon_vtbl1_v: {
8849 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
8851 case NEON::BI__builtin_neon_vtbl2_v: {
8853 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
8855 case NEON::BI__builtin_neon_vtbl3_v: {
8857 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
8859 case NEON::BI__builtin_neon_vtbl4_v: {
8861 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
8863 case NEON::BI__builtin_neon_vtbx1_v: {
8866 Intrinsic::aarch64_neon_tbl1,
"vtbl1");
8868 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
8869 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
8870 CmpRes = Builder.CreateSExt(CmpRes, Ty);
8872 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
8873 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
8874 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
8876 case NEON::BI__builtin_neon_vtbx2_v: {
8878 Ty, Intrinsic::aarch64_neon_tbx1,
"vtbx1");
8880 case NEON::BI__builtin_neon_vtbx3_v: {
8883 Intrinsic::aarch64_neon_tbl2,
"vtbl2");
8885 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
8886 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
8888 CmpRes = Builder.CreateSExt(CmpRes, Ty);
8890 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
8891 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
8892 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
8894 case NEON::BI__builtin_neon_vtbx4_v: {
8896 Ty, Intrinsic::aarch64_neon_tbx2,
"vtbx2");
8898 case NEON::BI__builtin_neon_vqtbl1_v:
8899 case NEON::BI__builtin_neon_vqtbl1q_v:
8900 Int = Intrinsic::aarch64_neon_tbl1;
s =
"vtbl1";
break;
8901 case NEON::BI__builtin_neon_vqtbl2_v:
8902 case NEON::BI__builtin_neon_vqtbl2q_v: {
8903 Int = Intrinsic::aarch64_neon_tbl2;
s =
"vtbl2";
break;
8904 case NEON::BI__builtin_neon_vqtbl3_v:
8905 case NEON::BI__builtin_neon_vqtbl3q_v:
8906 Int = Intrinsic::aarch64_neon_tbl3;
s =
"vtbl3";
break;
8907 case NEON::BI__builtin_neon_vqtbl4_v:
8908 case NEON::BI__builtin_neon_vqtbl4q_v:
8909 Int = Intrinsic::aarch64_neon_tbl4;
s =
"vtbl4";
break;
8910 case NEON::BI__builtin_neon_vqtbx1_v:
8911 case NEON::BI__builtin_neon_vqtbx1q_v:
8912 Int = Intrinsic::aarch64_neon_tbx1;
s =
"vtbx1";
break;
8913 case NEON::BI__builtin_neon_vqtbx2_v:
8914 case NEON::BI__builtin_neon_vqtbx2q_v:
8915 Int = Intrinsic::aarch64_neon_tbx2;
s =
"vtbx2";
break;
8916 case NEON::BI__builtin_neon_vqtbx3_v:
8917 case NEON::BI__builtin_neon_vqtbx3q_v:
8918 Int = Intrinsic::aarch64_neon_tbx3;
s =
"vtbx3";
break;
8919 case NEON::BI__builtin_neon_vqtbx4_v:
8920 case NEON::BI__builtin_neon_vqtbx4q_v:
8921 Int = Intrinsic::aarch64_neon_tbx4;
s =
"vtbx4";
break;
8933 auto *VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
8934 Op = Builder.CreateBitCast(Op,
Int16Ty);
8935 Value *
V = PoisonValue::get(VTy);
8936 llvm::Constant *CI = ConstantInt::get(
SizeTy, 0);
8937 Op = Builder.CreateInsertElement(
V, Op, CI);
8946 case SVETypeFlags::MemEltTyDefault:
8948 case SVETypeFlags::MemEltTyInt8:
8949 return Builder.getInt8Ty();
8950 case SVETypeFlags::MemEltTyInt16:
8951 return Builder.getInt16Ty();
8952 case SVETypeFlags::MemEltTyInt32:
8953 return Builder.getInt32Ty();
8954 case SVETypeFlags::MemEltTyInt64:
8955 return Builder.getInt64Ty();
8957 llvm_unreachable(
"Unknown MemEltType");
8963 llvm_unreachable(
"Invalid SVETypeFlag!");
8965 case SVETypeFlags::EltTyInt8:
8966 return Builder.getInt8Ty();
8967 case SVETypeFlags::EltTyInt16:
8968 return Builder.getInt16Ty();
8969 case SVETypeFlags::EltTyInt32:
8970 return Builder.getInt32Ty();
8971 case SVETypeFlags::EltTyInt64:
8972 return Builder.getInt64Ty();
8973 case SVETypeFlags::EltTyInt128:
8974 return Builder.getInt128Ty();
8976 case SVETypeFlags::EltTyFloat16:
8977 return Builder.getHalfTy();
8978 case SVETypeFlags::EltTyFloat32:
8979 return Builder.getFloatTy();
8980 case SVETypeFlags::EltTyFloat64:
8981 return Builder.getDoubleTy();
8983 case SVETypeFlags::EltTyBFloat16:
8984 return Builder.getBFloatTy();
8986 case SVETypeFlags::EltTyBool8:
8987 case SVETypeFlags::EltTyBool16:
8988 case SVETypeFlags::EltTyBool32:
8989 case SVETypeFlags::EltTyBool64:
8990 return Builder.getInt1Ty();
8996llvm::ScalableVectorType *
8999 default: llvm_unreachable(
"Unhandled SVETypeFlag!");
9001 case SVETypeFlags::EltTyInt8:
9002 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9003 case SVETypeFlags::EltTyInt16:
9004 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9005 case SVETypeFlags::EltTyInt32:
9006 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9007 case SVETypeFlags::EltTyInt64:
9008 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9010 case SVETypeFlags::EltTyBFloat16:
9011 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9012 case SVETypeFlags::EltTyFloat16:
9013 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9014 case SVETypeFlags::EltTyFloat32:
9015 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9016 case SVETypeFlags::EltTyFloat64:
9017 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9019 case SVETypeFlags::EltTyBool8:
9020 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9021 case SVETypeFlags::EltTyBool16:
9022 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9023 case SVETypeFlags::EltTyBool32:
9024 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9025 case SVETypeFlags::EltTyBool64:
9026 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9031llvm::ScalableVectorType *
9035 llvm_unreachable(
"Invalid SVETypeFlag!");
9037 case SVETypeFlags::EltTyInt8:
9038 return llvm::ScalableVectorType::get(Builder.getInt8Ty(), 16);
9039 case SVETypeFlags::EltTyInt16:
9040 return llvm::ScalableVectorType::get(Builder.getInt16Ty(), 8);
9041 case SVETypeFlags::EltTyInt32:
9042 return llvm::ScalableVectorType::get(Builder.getInt32Ty(), 4);
9043 case SVETypeFlags::EltTyInt64:
9044 return llvm::ScalableVectorType::get(Builder.getInt64Ty(), 2);
9046 case SVETypeFlags::EltTyFloat16:
9047 return llvm::ScalableVectorType::get(Builder.getHalfTy(), 8);
9048 case SVETypeFlags::EltTyBFloat16:
9049 return llvm::ScalableVectorType::get(Builder.getBFloatTy(), 8);
9050 case SVETypeFlags::EltTyFloat32:
9051 return llvm::ScalableVectorType::get(Builder.getFloatTy(), 4);
9052 case SVETypeFlags::EltTyFloat64:
9053 return llvm::ScalableVectorType::get(Builder.getDoubleTy(), 2);
9055 case SVETypeFlags::EltTyBool8:
9056 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
9057 case SVETypeFlags::EltTyBool16:
9058 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 8);
9059 case SVETypeFlags::EltTyBool32:
9060 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 4);
9061 case SVETypeFlags::EltTyBool64:
9062 return llvm::ScalableVectorType::get(Builder.getInt1Ty(), 2);
9070 return Builder.CreateCall(Ptrue, {Builder.getInt32( 31)});
9077 return llvm::ScalableVectorType::get(EltTy, NumElts);
9083 llvm::ScalableVectorType *VTy) {
9084 auto *RTy = llvm::VectorType::get(IntegerType::get(
getLLVMContext(), 1), VTy);
9089 llvm::Type *IntrinsicTy;
9090 switch (VTy->getMinNumElements()) {
9092 llvm_unreachable(
"unsupported element count!");
9097 IntID = Intrinsic::aarch64_sve_convert_from_svbool;
9101 IntID = Intrinsic::aarch64_sve_convert_to_svbool;
9102 IntrinsicTy = Pred->
getType();
9107 Value *
C = Builder.CreateCall(F, Pred);
9108 assert(
C->getType() == RTy &&
"Unexpected return type!");
9116 auto *OverloadedTy =
9126 Function *F =
nullptr;
9127 if (Ops[1]->
getType()->isVectorTy())
9142 if (Ops.size() == 2) {
9143 assert(Ops[1]->
getType()->isVectorTy() &&
"Scalar base requires an offset");
9144 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9149 if (!TypeFlags.
isByteIndexed() && Ops[1]->getType()->isVectorTy()) {
9150 unsigned BytesPerElt =
9151 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9152 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9155 Value *
Call = Builder.CreateCall(F, Ops);
9159 return TypeFlags.
isZExtReturn() ? Builder.CreateZExt(Call, ResultTy)
9160 : Builder.CreateSExt(Call, ResultTy);
9167 auto *OverloadedTy =
9172 Ops.insert(Ops.begin(), Ops.pop_back_val());
9174 Function *F =
nullptr;
9175 if (Ops[2]->
getType()->isVectorTy())
9190 if (Ops.size() == 3) {
9191 assert(Ops[1]->
getType()->isVectorTy() &&
"Scalar base requires an offset");
9192 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9197 Ops[0] = Builder.CreateTrunc(Ops[0], OverloadedTy);
9208 if (!TypeFlags.
isByteIndexed() && Ops[2]->getType()->isVectorTy()) {
9209 unsigned BytesPerElt =
9210 OverloadedTy->getElementType()->getScalarSizeInBits() / 8;
9211 Ops[3] = Builder.CreateShl(Ops[3], Log2_32(BytesPerElt));
9214 return Builder.CreateCall(F, Ops);
9222 auto *OverloadedTy = dyn_cast<llvm::ScalableVectorType>(Ops[1]->
getType());
9230 if (Ops[1]->
getType()->isVectorTy()) {
9231 if (Ops.size() == 3) {
9233 Ops.push_back(ConstantInt::get(
Int64Ty, 0));
9236 std::swap(Ops[2], Ops[3]);
9240 unsigned BytesPerElt = MemEltTy->getPrimitiveSizeInBits() / 8;
9241 if (BytesPerElt > 1)
9242 Ops[2] = Builder.CreateShl(Ops[2], Log2_32(BytesPerElt));
9247 return Builder.CreateCall(F, Ops);
9253 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9254 auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
9255 auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
9259 case Intrinsic::aarch64_sve_ld2_sret:
9262 case Intrinsic::aarch64_sve_ld3_sret:
9265 case Intrinsic::aarch64_sve_ld4_sret:
9269 llvm_unreachable(
"unknown intrinsic!");
9271 auto RetTy = llvm::VectorType::get(VTy->getElementType(),
9272 VTy->getElementCount() *
N);
9275 Value *BasePtr= Builder.CreateBitCast(Ops[1], VecPtrTy);
9279 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
9281 BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
9283 Value *
Call = Builder.CreateCall(F, {Predicate, BasePtr});
9284 unsigned MinElts = VTy->getMinNumElements();
9285 Value *
Ret = llvm::PoisonValue::get(RetTy);
9286 for (
unsigned I = 0; I <
N; I++) {
9288 Value *SRet = Builder.CreateExtractValue(Call, I);
9289 Ret = Builder.CreateInsertVector(RetTy, Ret, SRet, Idx);
9297 llvm::ScalableVectorType *VTy =
getSVEType(TypeFlags);
9298 auto VecPtrTy = llvm::PointerType::getUnqual(VTy);
9299 auto EltPtrTy = llvm::PointerType::getUnqual(VTy->getElementType());
9303 case Intrinsic::aarch64_sve_st2:
9306 case Intrinsic::aarch64_sve_st3:
9309 case Intrinsic::aarch64_sve_st4:
9313 llvm_unreachable(
"unknown intrinsic!");
9317 Value *BasePtr = Builder.CreateBitCast(Ops[1], VecPtrTy);
9321 BasePtr = Builder.CreateGEP(VTy, BasePtr, Ops[2]);
9323 BasePtr = Builder.CreateBitCast(BasePtr, EltPtrTy);
9324 Value *Val = Ops.back();
9329 unsigned MinElts = VTy->getElementCount().getKnownMinValue();
9330 for (
unsigned I = 0; I <
N; ++I) {
9332 Operands.push_back(Builder.CreateExtractVector(VTy, Val, Idx));
9334 Operands.append({Predicate, BasePtr});
9337 return Builder.CreateCall(F, Operands);
9345 unsigned BuiltinID) {
9354 Value *
Call = Builder.CreateCall(F, {Ops[0], Ops[1]});
9357 llvm::ScalableVectorType *Ty =
getSVEType(TypeFlags);
9363 llvm::Type *OverloadedTy =
getSVEType(TypeFlags);
9365 return Builder.CreateCall(F, {Ops[0], Builder.getInt32(0)});
9370 unsigned BuiltinID) {
9373 auto *MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9376 Value *BasePtr = Ops[1];
9380 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
9382 Value *PrfOp = Ops.back();
9385 return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp});
9389 llvm::Type *ReturnTy,
9392 bool IsZExtReturn) {
9400 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9403 Value *BasePtr = Ops[1];
9407 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
9415 return IsZExtReturn ? Builder.CreateZExt(Load, VectorTy)
9416 : Builder.CreateSExt(Load, VectorTy);
9421 unsigned BuiltinID) {
9429 auto MemoryTy = llvm::ScalableVectorType::get(MemEltTy, VectorTy);
9432 Value *BasePtr = Ops[1];
9435 if (Ops.size() == 4)
9436 BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Ops[2]);
9439 llvm::Value *Val = Builder.CreateTrunc(Ops.back(), MemoryTy);
9450 llvm::Value *CastOffset = Builder.CreateIntCast(
Offset,
Int32Ty,
false);
9451 return Builder.CreateAdd(
Base, CastOffset,
"tileslice");
9461 NewOps.push_back(Ops[3]);
9463 llvm::Value *BasePtr = Ops[4];
9467 if (Ops.size() == 6) {
9468 Function *StreamingVectorLength =
9470 llvm::Value *StreamingVectorLengthCall =
9471 Builder.CreateCall(StreamingVectorLength);
9472 llvm::Value *Mulvl =
9473 Builder.CreateMul(StreamingVectorLengthCall, Ops[5],
"mulvl");
9475 BasePtr = Builder.CreateGEP(
Int8Ty, Ops[4], Mulvl);
9477 NewOps.push_back(BasePtr);
9478 NewOps.push_back(Ops[0]);
9481 return Builder.CreateCall(F, NewOps);
9498 return Builder.CreateCall(F, Ops);
9505 if (Ops.size() == 0)
9506 Ops.push_back(llvm::ConstantInt::get(
Int32Ty, 255));
9508 return Builder.CreateCall(F, Ops);
9515 llvm::Value *CntsbCall = Builder.CreateCall(Cntsb, {},
"svlb");
9516 llvm::Value *MulVL = Builder.CreateMul(
9518 Builder.getInt64(cast<llvm::ConstantInt>(Ops[1])->getZExtValue()),
9520 Ops[2] = Builder.CreateGEP(
Int8Ty, Ops[2], MulVL);
9524 return Builder.CreateCall(F, Ops);
9530 return Builder.CreateVectorSplat(
9531 cast<llvm::VectorType>(Ty)->getElementCount(), Scalar);
9545 return Builder.CreateBitCast(Val, Ty);
9550 auto *SplatZero = Constant::getNullValue(Ty);
9551 Ops.insert(Ops.begin(), SplatZero);
9556 auto *SplatUndef = UndefValue::get(Ty);
9557 Ops.insert(Ops.begin(), SplatUndef);
9562 llvm::Type *ResultType,
9567 llvm::Type *DefaultType =
getSVEType(TypeFlags);
9570 return {DefaultType, Ops[1]->getType()};
9576 return {Ops[0]->getType(), Ops.back()->getType()};
9579 return {DefaultType};
9586 "Expects TypleFlag isTupleSet or TypeFlags.isTupleSet()");
9589 auto *SingleVecTy = dyn_cast<llvm::ScalableVectorType>(
9590 TypeFlags.
isTupleSet() ? Ops[2]->getType() : Ty);
9592 I * SingleVecTy->getMinNumElements());
9595 return Builder.CreateInsertVector(Ty, Ops[0], Ops[2], Idx);
9596 return Builder.CreateExtractVector(Ty, Ops[0], Idx);
9602 assert(TypeFlags.
isTupleCreate() &&
"Expects TypleFlag isTupleCreate");
9604 auto *SrcTy = dyn_cast<llvm::ScalableVectorType>(Ops[0]->
getType());
9605 unsigned MinElts = SrcTy->getMinNumElements();
9606 Value *
Call = llvm::PoisonValue::get(Ty);
9607 for (
unsigned I = 0; I < Ops.size(); I++) {
9609 Call = Builder.CreateInsertVector(Ty, Call, Ops[I], Idx);
9618 unsigned ICEArguments = 0;
9624 if (BuiltinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 &&
9625 BuiltinID <= SVE::BI__builtin_sve_reinterpret_f64_f64) {
9631 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
9632 if ((ICEArguments & (1 << i)) == 0)
9637 std::optional<llvm::APSInt>
Result =
9639 assert(
Result &&
"Expected argument to be a constant");
9674 return UndefValue::get(Ty);
9675 else if (Builtin->LLVMIntrinsic != 0) {
9676 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZeroExp)
9679 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeAnyExp)
9685 Ops.push_back(Builder.getInt32( 31));
9687 Ops.insert(&Ops[1], Builder.getInt32( 31));
9690 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
9691 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->
getType()))
9692 if (PredTy->getElementType()->isIntegerTy(1))
9702 std::swap(Ops[1], Ops[2]);
9704 std::swap(Ops[1], Ops[2]);
9707 std::swap(Ops[1], Ops[2]);
9710 std::swap(Ops[1], Ops[3]);
9713 if (TypeFlags.
getMergeType() == SVETypeFlags::MergeZero) {
9714 llvm::Type *OpndTy = Ops[1]->getType();
9715 auto *SplatZero = Constant::getNullValue(OpndTy);
9716 Ops[1] = Builder.CreateSelect(Ops[0], Ops[1], SplatZero);
9721 Value *
Call = Builder.CreateCall(F, Ops);
9724 if (
auto PredTy = dyn_cast<llvm::VectorType>(
Call->getType()))
9725 if (PredTy->getScalarType()->isIntegerTy(1))
9731 switch (BuiltinID) {
9735 case SVE::BI__builtin_sve_svmov_b_z: {
9738 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
9739 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_sve_and_z, OverloadedTy);
9740 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[1]});
9743 case SVE::BI__builtin_sve_svnot_b_z: {
9746 llvm::Type* OverloadedTy =
getSVEType(TypeFlags);
9747 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_sve_eor_z, OverloadedTy);
9748 return Builder.CreateCall(F, {Ops[0], Ops[1], Ops[0]});
9751 case SVE::BI__builtin_sve_svmovlb_u16:
9752 case SVE::BI__builtin_sve_svmovlb_u32:
9753 case SVE::BI__builtin_sve_svmovlb_u64:
9754 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllb);
9756 case SVE::BI__builtin_sve_svmovlb_s16:
9757 case SVE::BI__builtin_sve_svmovlb_s32:
9758 case SVE::BI__builtin_sve_svmovlb_s64:
9759 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllb);
9761 case SVE::BI__builtin_sve_svmovlt_u16:
9762 case SVE::BI__builtin_sve_svmovlt_u32:
9763 case SVE::BI__builtin_sve_svmovlt_u64:
9764 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_ushllt);
9766 case SVE::BI__builtin_sve_svmovlt_s16:
9767 case SVE::BI__builtin_sve_svmovlt_s32:
9768 case SVE::BI__builtin_sve_svmovlt_s64:
9769 return EmitSVEMovl(TypeFlags, Ops, Intrinsic::aarch64_sve_sshllt);
9771 case SVE::BI__builtin_sve_svpmullt_u16:
9772 case SVE::BI__builtin_sve_svpmullt_u64:
9773 case SVE::BI__builtin_sve_svpmullt_n_u16:
9774 case SVE::BI__builtin_sve_svpmullt_n_u64:
9775 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullt_pair);
9777 case SVE::BI__builtin_sve_svpmullb_u16:
9778 case SVE::BI__builtin_sve_svpmullb_u64:
9779 case SVE::BI__builtin_sve_svpmullb_n_u16:
9780 case SVE::BI__builtin_sve_svpmullb_n_u64:
9781 return EmitSVEPMull(TypeFlags, Ops, Intrinsic::aarch64_sve_pmullb_pair);
9783 case SVE::BI__builtin_sve_svdup_n_b8:
9784 case SVE::BI__builtin_sve_svdup_n_b16:
9785 case SVE::BI__builtin_sve_svdup_n_b32:
9786 case SVE::BI__builtin_sve_svdup_n_b64: {
9788 Builder.CreateICmpNE(Ops[0], Constant::getNullValue(Ops[0]->
getType()));
9789 llvm::ScalableVectorType *OverloadedTy =
getSVEType(TypeFlags);
9794 case SVE::BI__builtin_sve_svdupq_n_b8:
9795 case SVE::BI__builtin_sve_svdupq_n_b16:
9796 case SVE::BI__builtin_sve_svdupq_n_b32:
9797 case SVE::BI__builtin_sve_svdupq_n_b64:
9798 case SVE::BI__builtin_sve_svdupq_n_u8:
9799 case SVE::BI__builtin_sve_svdupq_n_s8:
9800 case SVE::BI__builtin_sve_svdupq_n_u64:
9801 case SVE::BI__builtin_sve_svdupq_n_f64:
9802 case SVE::BI__builtin_sve_svdupq_n_s64:
9803 case SVE::BI__builtin_sve_svdupq_n_u16:
9804 case SVE::BI__builtin_sve_svdupq_n_f16:
9805 case SVE::BI__builtin_sve_svdupq_n_bf16:
9806 case SVE::BI__builtin_sve_svdupq_n_s16:
9807 case SVE::BI__builtin_sve_svdupq_n_u32:
9808 case SVE::BI__builtin_sve_svdupq_n_f32:
9809 case SVE::BI__builtin_sve_svdupq_n_s32: {
9812 unsigned NumOpnds = Ops.size();
9820 llvm::Type *EltTy = Ops[0]->getType();
9825 for (
unsigned I = 0; I < NumOpnds; ++I)
9826 VecOps.push_back(Builder.CreateZExt(Ops[I], EltTy));
9830 Value *InsertSubVec = Builder.CreateInsertVector(
9831 OverloadedTy, PoisonValue::get(OverloadedTy), Vec, Builder.getInt64(0));
9836 Builder.CreateCall(F, {InsertSubVec, Builder.getInt64(0)});
9846 : Intrinsic::aarch64_sve_cmpne_wide,
9849 F, {Pred, DupQLane,
EmitSVEDupX(Builder.getInt64(0))});
9853 case SVE::BI__builtin_sve_svpfalse_b:
9854 return ConstantInt::getFalse(Ty);
9856 case SVE::BI__builtin_sve_svlen_bf16:
9857 case SVE::BI__builtin_sve_svlen_f16:
9858 case SVE::BI__builtin_sve_svlen_f32:
9859 case SVE::BI__builtin_sve_svlen_f64:
9860 case SVE::BI__builtin_sve_svlen_s8:
9861 case SVE::BI__builtin_sve_svlen_s16:
9862 case SVE::BI__builtin_sve_svlen_s32:
9863 case SVE::BI__builtin_sve_svlen_s64:
9864 case SVE::BI__builtin_sve_svlen_u8:
9865 case SVE::BI__builtin_sve_svlen_u16:
9866 case SVE::BI__builtin_sve_svlen_u32:
9867 case SVE::BI__builtin_sve_svlen_u64: {
9871 llvm::ConstantInt::get(Ty, VTy->getElementCount().getKnownMinValue());
9874 return Builder.CreateMul(NumEls, Builder.CreateCall(F));
9877 case SVE::BI__builtin_sve_svtbl2_u8:
9878 case SVE::BI__builtin_sve_svtbl2_s8:
9879 case SVE::BI__builtin_sve_svtbl2_u16:
9880 case SVE::BI__builtin_sve_svtbl2_s16:
9881 case SVE::BI__builtin_sve_svtbl2_u32:
9882 case SVE::BI__builtin_sve_svtbl2_s32:
9883 case SVE::BI__builtin_sve_svtbl2_u64:
9884 case SVE::BI__builtin_sve_svtbl2_s64:
9885 case SVE::BI__builtin_sve_svtbl2_f16:
9886 case SVE::BI__builtin_sve_svtbl2_bf16:
9887 case SVE::BI__builtin_sve_svtbl2_f32:
9888 case SVE::BI__builtin_sve_svtbl2_f64: {
9891 Value *V0 = Builder.CreateExtractVector(VTy, Ops[0],
9893 unsigned MinElts = VTy->getMinNumElements();
9894 Value *V1 = Builder.CreateExtractVector(
9895 VTy, Ops[0], ConstantInt::get(
CGM.
Int64Ty, MinElts));
9897 return Builder.CreateCall(F, {V0, V1, Ops[1]});
9900 case SVE::BI__builtin_sve_svset_neonq_s8:
9901 case SVE::BI__builtin_sve_svset_neonq_s16:
9902 case SVE::BI__builtin_sve_svset_neonq_s32:
9903 case SVE::BI__builtin_sve_svset_neonq_s64:
9904 case SVE::BI__builtin_sve_svset_neonq_u8:
9905 case SVE::BI__builtin_sve_svset_neonq_u16:
9906 case SVE::BI__builtin_sve_svset_neonq_u32:
9907 case SVE::BI__builtin_sve_svset_neonq_u64:
9908 case SVE::BI__builtin_sve_svset_neonq_f16:
9909 case SVE::BI__builtin_sve_svset_neonq_f32:
9910 case SVE::BI__builtin_sve_svset_neonq_f64:
9911 case SVE::BI__builtin_sve_svset_neonq_bf16: {
9912 return Builder.CreateInsertVector(Ty, Ops[0], Ops[1], Builder.getInt64(0));
9915 case SVE::BI__builtin_sve_svget_neonq_s8:
9916 case SVE::BI__builtin_sve_svget_neonq_s16:
9917 case SVE::BI__builtin_sve_svget_neonq_s32:
9918 case SVE::BI__builtin_sve_svget_neonq_s64:
9919 case SVE::BI__builtin_sve_svget_neonq_u8:
9920 case SVE::BI__builtin_sve_svget_neonq_u16:
9921 case SVE::BI__builtin_sve_svget_neonq_u32:
9922 case SVE::BI__builtin_sve_svget_neonq_u64:
9923 case SVE::BI__builtin_sve_svget_neonq_f16:
9924 case SVE::BI__builtin_sve_svget_neonq_f32:
9925 case SVE::BI__builtin_sve_svget_neonq_f64:
9926 case SVE::BI__builtin_sve_svget_neonq_bf16: {
9927 return Builder.CreateExtractVector(Ty, Ops[0], Builder.getInt64(0));
9930 case SVE::BI__builtin_sve_svdup_neonq_s8:
9931 case SVE::BI__builtin_sve_svdup_neonq_s16:
9932 case SVE::BI__builtin_sve_svdup_neonq_s32:
9933 case SVE::BI__builtin_sve_svdup_neonq_s64:
9934 case SVE::BI__builtin_sve_svdup_neonq_u8:
9935 case SVE::BI__builtin_sve_svdup_neonq_u16:
9936 case SVE::BI__builtin_sve_svdup_neonq_u32:
9937 case SVE::BI__builtin_sve_svdup_neonq_u64:
9938 case SVE::BI__builtin_sve_svdup_neonq_f16:
9939 case SVE::BI__builtin_sve_svdup_neonq_f32:
9940 case SVE::BI__builtin_sve_svdup_neonq_f64:
9941 case SVE::BI__builtin_sve_svdup_neonq_bf16: {
9942 Value *
Insert = Builder.CreateInsertVector(Ty, PoisonValue::get(Ty), Ops[0],
9943 Builder.getInt64(0));
9944 return Builder.CreateIntrinsic(Intrinsic::aarch64_sve_dupq_lane, {Ty},
9945 {
Insert, Builder.getInt64(0)});
9956 unsigned ICEArguments = 0;
9963 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
9964 if ((ICEArguments & (1 << i)) == 0)
9969 std::optional<llvm::APSInt>
Result =
9971 assert(
Result &&
"Expected argument to be a constant");
9985 return EmitSMELd1St1(TypeFlags, Ops, Builtin->LLVMIntrinsic);
9988 else if (BuiltinID == SME::BI__builtin_sme_svzero_mask_za ||
9989 BuiltinID == SME::BI__builtin_sme_svzero_za)
9990 return EmitSMEZero(TypeFlags, Ops, Builtin->LLVMIntrinsic);
9991 else if (BuiltinID == SME::BI__builtin_sme_svldr_vnum_za ||
9992 BuiltinID == SME::BI__builtin_sme_svstr_vnum_za)
9993 return EmitSMELdrStr(TypeFlags, Ops, Builtin->LLVMIntrinsic);
9994 else if (Builtin->LLVMIntrinsic != 0) {
9996 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
9997 if (
auto PredTy = dyn_cast<llvm::VectorType>(Ops[i]->
getType()))
9998 if (PredTy->getElementType()->isIntegerTy(1))
10003 Value *
Call = Builder.CreateCall(F, Ops);
10013 llvm::Triple::ArchType Arch) {
10022 unsigned HintID =
static_cast<unsigned>(-1);
10023 switch (BuiltinID) {
10025 case clang::AArch64::BI__builtin_arm_nop:
10028 case clang::AArch64::BI__builtin_arm_yield:
10029 case clang::AArch64::BI__yield:
10032 case clang::AArch64::BI__builtin_arm_wfe:
10033 case clang::AArch64::BI__wfe:
10036 case clang::AArch64::BI__builtin_arm_wfi:
10037 case clang::AArch64::BI__wfi:
10040 case clang::AArch64::BI__builtin_arm_sev:
10041 case clang::AArch64::BI__sev:
10044 case clang::AArch64::BI__builtin_arm_sevl:
10045 case clang::AArch64::BI__sevl:
10050 if (HintID !=
static_cast<unsigned>(-1)) {
10052 return Builder.CreateCall(F, llvm::ConstantInt::get(
Int32Ty, HintID));
10055 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit) {
10057 "rbit of unusual size!");
10059 return Builder.CreateCall(
10060 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10062 if (BuiltinID == clang::AArch64::BI__builtin_arm_rbit64) {
10064 "rbit of unusual size!");
10066 return Builder.CreateCall(
10067 CGM.
getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
10070 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz ||
10071 BuiltinID == clang::AArch64::BI__builtin_arm_clz64) {
10074 Value *Res = Builder.CreateCall(F, {Arg, Builder.getInt1(
false)});
10075 if (BuiltinID == clang::AArch64::BI__builtin_arm_clz64)
10076 Res = Builder.CreateTrunc(Res, Builder.getInt32Ty());
10080 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls) {
10082 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_cls), Arg,
10085 if (BuiltinID == clang::AArch64::BI__builtin_arm_cls64) {
10087 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_cls64), Arg,
10091 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32zf ||
10092 BuiltinID == clang::AArch64::BI__builtin_arm_rint32z) {
10094 llvm::Type *Ty = Arg->getType();
10095 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint32z, Ty),
10099 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64zf ||
10100 BuiltinID == clang::AArch64::BI__builtin_arm_rint64z) {
10102 llvm::Type *Ty = Arg->getType();
10103 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint64z, Ty),
10107 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint32xf ||
10108 BuiltinID == clang::AArch64::BI__builtin_arm_rint32x) {
10110 llvm::Type *Ty = Arg->getType();
10111 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint32x, Ty),
10115 if (BuiltinID == clang::AArch64::BI__builtin_arm_rint64xf ||
10116 BuiltinID == clang::AArch64::BI__builtin_arm_rint64x) {
10118 llvm::Type *Ty = Arg->getType();
10119 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::aarch64_frint64x, Ty),
10123 if (BuiltinID == clang::AArch64::BI__builtin_arm_jcvt) {
10125 "__jcvt of unusual size!");
10127 return Builder.CreateCall(
10131 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b ||
10132 BuiltinID == clang::AArch64::BI__builtin_arm_st64b ||
10133 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv ||
10134 BuiltinID == clang::AArch64::BI__builtin_arm_st64bv0) {
10138 if (BuiltinID == clang::AArch64::BI__builtin_arm_ld64b) {
10142 llvm::Value *Val = Builder.CreateCall(F, MemAddr);
10143 llvm::Value *ToRet;
10144 for (
size_t i = 0; i < 8; i++) {
10145 llvm::Value *ValOffsetPtr =
10146 Builder.CreateGEP(
Int64Ty, ValPtr, Builder.getInt32(i));
10149 ToRet = Builder.CreateStore(Builder.CreateExtractValue(Val, i), Addr);
10156 Args.push_back(MemAddr);
10157 for (
size_t i = 0; i < 8; i++) {
10158 llvm::Value *ValOffsetPtr =
10159 Builder.CreateGEP(
Int64Ty, ValPtr, Builder.getInt32(i));
10162 Args.push_back(Builder.CreateLoad(Addr));
10165 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_st64b
10166 ? Intrinsic::aarch64_st64b
10167 : BuiltinID == clang::AArch64::BI__builtin_arm_st64bv
10168 ? Intrinsic::aarch64_st64bv
10169 : Intrinsic::aarch64_st64bv0);
10171 return Builder.CreateCall(F, Args);
10175 if (BuiltinID == clang::AArch64::BI__builtin_arm_rndr ||
10176 BuiltinID == clang::AArch64::BI__builtin_arm_rndrrs) {
10178 auto Intr = (BuiltinID == clang::AArch64::BI__builtin_arm_rndr
10179 ? Intrinsic::aarch64_rndr
10180 : Intrinsic::aarch64_rndrrs);
10182 llvm::Value *Val = Builder.CreateCall(F);
10183 Value *RandomValue = Builder.CreateExtractValue(Val, 0);
10184 Value *Status = Builder.CreateExtractValue(Val, 1);
10187 Builder.CreateStore(RandomValue, MemAddress);
10188 Status = Builder.CreateZExt(Status,
Int32Ty);
10192 if (BuiltinID == clang::AArch64::BI__clear_cache) {
10193 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
10196 for (
unsigned i = 0; i < 2; i++)
10200 StringRef Name = FD->
getName();
10204 if ((BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
10205 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) &&
10209 ? Intrinsic::aarch64_ldaxp
10210 : Intrinsic::aarch64_ldxp);
10213 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr,
Int8PtrTy),
10216 Value *Val0 = Builder.CreateExtractValue(Val, 1);
10217 Value *Val1 = Builder.CreateExtractValue(Val, 0);
10218 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
10219 Val0 = Builder.CreateZExt(Val0, Int128Ty);
10220 Val1 = Builder.CreateZExt(Val1, Int128Ty);
10222 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
10223 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
10224 Val = Builder.CreateOr(Val, Val1);
10226 }
else if (BuiltinID == clang::AArch64::BI__builtin_arm_ldrex ||
10227 BuiltinID == clang::AArch64::BI__builtin_arm_ldaex) {
10232 llvm::Type *
IntTy =
10234 llvm::Type *PtrTy = llvm::PointerType::getUnqual(
getLLVMContext());
10238 ? Intrinsic::aarch64_ldaxr
10239 : Intrinsic::aarch64_ldxr,
10241 CallInst *Val = Builder.CreateCall(F, LoadAddr,
"ldxr");
10245 if (RealResTy->isPointerTy())
10246 return Builder.CreateIntToPtr(Val, RealResTy);
10248 llvm::Type *IntResTy = llvm::IntegerType::get(
10250 return Builder.CreateBitCast(Builder.CreateTruncOrBitCast(Val, IntResTy),
10254 if ((BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
10255 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) &&
10259 ? Intrinsic::aarch64_stlxp
10260 : Intrinsic::aarch64_stxp);
10267 llvm::Value *Val = Builder.CreateLoad(Tmp);
10269 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
10270 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
10273 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
10276 if (BuiltinID == clang::AArch64::BI__builtin_arm_strex ||
10277 BuiltinID == clang::AArch64::BI__builtin_arm_stlex) {
10282 llvm::Type *StoreTy =
10285 if (StoreVal->
getType()->isPointerTy())
10286 StoreVal = Builder.CreatePtrToInt(StoreVal,
Int64Ty);
10288 llvm::Type *
IntTy = llvm::IntegerType::get(
10291 StoreVal = Builder.CreateBitCast(StoreVal,
IntTy);
10292 StoreVal = Builder.CreateZExtOrBitCast(StoreVal,
Int64Ty);
10297 ? Intrinsic::aarch64_stlxr
10298 : Intrinsic::aarch64_stxr,
10300 CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
10302 1, Attribute::get(
getLLVMContext(), Attribute::ElementType, StoreTy));
10306 if (BuiltinID == clang::AArch64::BI__getReg) {
10309 llvm_unreachable(
"Sema will ensure that the parameter is constant");
10315 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
10316 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
10317 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
10319 llvm::Function *F =
10321 return Builder.CreateCall(F, Metadata);
10324 if (BuiltinID == clang::AArch64::BI__break) {
10327 llvm_unreachable(
"Sema will ensure that the parameter is constant");
10329 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::aarch64_break);
10333 if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
10335 return Builder.CreateCall(F);
10338 if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
10339 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
10340 llvm::SyncScope::SingleThread);
10343 Intrinsic::ID CRCIntrinsicID = Intrinsic::not_intrinsic;
10344 switch (BuiltinID) {
10345 case clang::AArch64::BI__builtin_arm_crc32b:
10346 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
10347 case clang::AArch64::BI__builtin_arm_crc32cb:
10348 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
10349 case clang::AArch64::BI__builtin_arm_crc32h:
10350 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
10351 case clang::AArch64::BI__builtin_arm_crc32ch:
10352 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
10353 case clang::AArch64::BI__builtin_arm_crc32w:
10354 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
10355 case clang::AArch64::BI__builtin_arm_crc32cw:
10356 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
10357 case clang::AArch64::BI__builtin_arm_crc32d:
10358 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
10359 case clang::AArch64::BI__builtin_arm_crc32cd:
10360 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
10363 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
10368 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
10369 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
10371 return Builder.CreateCall(F, {Arg0, Arg1});
10375 if (BuiltinID == AArch64::BI__builtin_arm_mops_memset_tag) {
10379 Dst = Builder.CreatePointerCast(Dst,
Int8PtrTy);
10380 Val = Builder.CreateTrunc(Val,
Int8Ty);
10381 Size = Builder.CreateIntCast(Size,
Int64Ty,
false);
10382 return Builder.CreateCall(
10383 CGM.
getIntrinsic(Intrinsic::aarch64_mops_memset_tag), {Dst, Val, Size});
10387 Intrinsic::ID MTEIntrinsicID = Intrinsic::not_intrinsic;
10388 switch (BuiltinID) {
10389 case clang::AArch64::BI__builtin_arm_irg:
10390 MTEIntrinsicID = Intrinsic::aarch64_irg;
break;
10391 case clang::AArch64::BI__builtin_arm_addg:
10392 MTEIntrinsicID = Intrinsic::aarch64_addg;
break;
10393 case clang::AArch64::BI__builtin_arm_gmi:
10394 MTEIntrinsicID = Intrinsic::aarch64_gmi;
break;
10395 case clang::AArch64::BI__builtin_arm_ldg:
10396 MTEIntrinsicID = Intrinsic::aarch64_ldg;
break;
10397 case clang::AArch64::BI__builtin_arm_stg:
10398 MTEIntrinsicID = Intrinsic::aarch64_stg;
break;
10399 case clang::AArch64::BI__builtin_arm_subp:
10400 MTEIntrinsicID = Intrinsic::aarch64_subp;
break;
10403 if (MTEIntrinsicID != Intrinsic::not_intrinsic) {
10406 if (MTEIntrinsicID == Intrinsic::aarch64_irg) {
10411 Mask = Builder.CreateZExt(Mask,
Int64Ty);
10412 Value *RV = Builder.CreateCall(
10414 return Builder.CreatePointerCast(RV, T);
10416 if (MTEIntrinsicID == Intrinsic::aarch64_addg) {
10421 TagOffset = Builder.CreateZExt(TagOffset,
Int64Ty);
10422 Value *RV = Builder.CreateCall(
10424 return Builder.CreatePointerCast(RV, T);
10426 if (MTEIntrinsicID == Intrinsic::aarch64_gmi) {
10430 ExcludedMask = Builder.CreateZExt(ExcludedMask,
Int64Ty);
10432 return Builder.CreateCall(
10438 if (MTEIntrinsicID == Intrinsic::aarch64_ldg) {
10440 TagAddress = Builder.CreatePointerCast(TagAddress,
Int8PtrTy);
10441 Value *RV = Builder.CreateCall(
10443 return Builder.CreatePointerCast(RV, T);
10448 if (MTEIntrinsicID == Intrinsic::aarch64_stg) {
10450 TagAddress = Builder.CreatePointerCast(TagAddress,
Int8PtrTy);
10451 return Builder.CreateCall(
10454 if (MTEIntrinsicID == Intrinsic::aarch64_subp) {
10457 PointerA = Builder.CreatePointerCast(PointerA,
Int8PtrTy);
10458 PointerB = Builder.CreatePointerCast(PointerB,
Int8PtrTy);
10459 return Builder.CreateCall(
10464 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
10465 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
10466 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
10467 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
10468 BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
10469 BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
10470 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
10471 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
10474 if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
10475 BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
10476 BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
10477 BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
10480 bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
10481 BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
10483 bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
10484 BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
10486 bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
10487 BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
10489 llvm::Type *ValueType;
10490 llvm::Type *RegisterType =
Int64Ty;
10493 }
else if (Is128Bit) {
10494 llvm::Type *Int128Ty =
10496 ValueType = Int128Ty;
10497 RegisterType = Int128Ty;
10498 }
else if (IsPointerBuiltin) {
10508 if (BuiltinID == clang::AArch64::BI_ReadStatusReg ||
10509 BuiltinID == clang::AArch64::BI_WriteStatusReg) {
10515 std::string SysRegStr;
10516 llvm::raw_string_ostream(SysRegStr) <<
10517 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
10518 ((SysReg >> 11) & 7) <<
":" <<
10519 ((SysReg >> 7) & 15) <<
":" <<
10520 ((SysReg >> 3) & 15) <<
":" <<
10523 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
10524 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
10525 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
10527 llvm::Type *RegisterType =
Int64Ty;
10528 llvm::Type *Types[] = { RegisterType };
10530 if (BuiltinID == clang::AArch64::BI_ReadStatusReg) {
10531 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::read_register, Types);
10533 return Builder.CreateCall(F, Metadata);
10536 llvm::Function *F =
CGM.
getIntrinsic(llvm::Intrinsic::write_register, Types);
10539 return Builder.CreateCall(F, { Metadata, ArgValue });
10542 if (BuiltinID == clang::AArch64::BI_AddressOfReturnAddress) {
10543 llvm::Function *F =
10545 return Builder.CreateCall(F);
10548 if (BuiltinID == clang::AArch64::BI__builtin_sponentry) {
10550 return Builder.CreateCall(F);
10553 if (BuiltinID == clang::AArch64::BI__mulh ||
10554 BuiltinID == clang::AArch64::BI__umulh) {
10556 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
10558 bool IsSigned = BuiltinID == clang::AArch64::BI__mulh;
10564 Value *MulResult, *HigherBits;
10566 MulResult = Builder.CreateNSWMul(LHS, RHS);
10567 HigherBits = Builder.CreateAShr(MulResult, 64);
10569 MulResult = Builder.CreateNUWMul(LHS, RHS);
10570 HigherBits = Builder.CreateLShr(MulResult, 64);
10572 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
10577 if (BuiltinID == AArch64::BI__writex18byte ||
10578 BuiltinID == AArch64::BI__writex18word ||
10579 BuiltinID == AArch64::BI__writex18dword ||
10580 BuiltinID == AArch64::BI__writex18qword) {
10585 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
10586 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
10587 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
10588 llvm::Function *F =
10590 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
10591 X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(
Int8Ty, 0));
10596 Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(
IntTy, 0));
10602 if (BuiltinID == AArch64::BI__readx18byte ||
10603 BuiltinID == AArch64::BI__readx18word ||
10604 BuiltinID == AArch64::BI__readx18dword ||
10605 BuiltinID == AArch64::BI__readx18qword) {
10610 llvm::Metadata *Ops[] = {llvm::MDString::get(Context,
"x18")};
10611 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
10612 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
10613 llvm::Function *F =
10615 llvm::Value *X18 = Builder.CreateCall(F, Metadata);
10616 X18 = Builder.CreateIntToPtr(X18, llvm::PointerType::get(
Int8Ty, 0));
10621 Ptr = Builder.CreatePointerCast(Ptr, llvm::PointerType::get(
IntTy, 0));
10628 if (std::optional<MSVCIntrin> MsvcIntId =
10634 return P.first == BuiltinID;
10637 BuiltinID = It->second;
10641 unsigned ICEArguments = 0;
10648 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
10650 switch (BuiltinID) {
10651 case NEON::BI__builtin_neon_vld1_v:
10652 case NEON::BI__builtin_neon_vld1q_v:
10653 case NEON::BI__builtin_neon_vld1_dup_v:
10654 case NEON::BI__builtin_neon_vld1q_dup_v:
10655 case NEON::BI__builtin_neon_vld1_lane_v:
10656 case NEON::BI__builtin_neon_vld1q_lane_v:
10657 case NEON::BI__builtin_neon_vst1_v:
10658 case NEON::BI__builtin_neon_vst1q_v:
10659 case NEON::BI__builtin_neon_vst1_lane_v:
10660 case NEON::BI__builtin_neon_vst1q_lane_v:
10661 case NEON::BI__builtin_neon_vldap1_lane_s64:
10662 case NEON::BI__builtin_neon_vldap1q_lane_s64:
10663 case NEON::BI__builtin_neon_vstl1_lane_s64:
10664 case NEON::BI__builtin_neon_vstl1q_lane_s64:
10672 if ((ICEArguments & (1 << i)) == 0) {
10677 Ops.push_back(llvm::ConstantInt::get(
10690 assert(
Result &&
"SISD intrinsic should have been handled");
10696 if (std::optional<llvm::APSInt>
Result =
10701 bool usgn =
Type.isUnsigned();
10702 bool quad =
Type.isQuad();
10705 switch (BuiltinID) {
10707 case NEON::BI__builtin_neon_vabsh_f16:
10710 case NEON::BI__builtin_neon_vaddq_p128: {
10713 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
10714 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
10715 Ops[0] = Builder.CreateXor(Ops[0], Ops[1]);
10716 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
10717 return Builder.CreateBitCast(Ops[0], Int128Ty);
10719 case NEON::BI__builtin_neon_vldrq_p128: {
10720 llvm::Type *Int128Ty = llvm::Type::getIntNTy(
getLLVMContext(), 128);
10721 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
10723 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
10726 case NEON::BI__builtin_neon_vstrq_p128: {
10727 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(
getLLVMContext(), 128);
10728 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
10731 case NEON::BI__builtin_neon_vcvts_f32_u32:
10732 case NEON::BI__builtin_neon_vcvtd_f64_u64:
10735 case NEON::BI__builtin_neon_vcvts_f32_s32:
10736 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
10738 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
10741 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
10743 return Builder.CreateUIToFP(Ops[0], FTy);
10744 return Builder.CreateSIToFP(Ops[0], FTy);
10746 case NEON::BI__builtin_neon_vcvth_f16_u16:
10747 case NEON::BI__builtin_neon_vcvth_f16_u32:
10748 case NEON::BI__builtin_neon_vcvth_f16_u64:
10751 case NEON::BI__builtin_neon_vcvth_f16_s16:
10752 case NEON::BI__builtin_neon_vcvth_f16_s32:
10753 case NEON::BI__builtin_neon_vcvth_f16_s64: {
10755 llvm::Type *FTy =
HalfTy;
10757 if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 64)
10759 else if (Ops[0]->
getType()->getPrimitiveSizeInBits() == 32)
10763 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
10765 return Builder.CreateUIToFP(Ops[0], FTy);
10766 return Builder.CreateSIToFP(Ops[0], FTy);
10768 case NEON::BI__builtin_neon_vcvtah_u16_f16:
10769 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
10770 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
10771 case NEON::BI__builtin_neon_vcvtph_u16_f16:
10772 case NEON::BI__builtin_neon_vcvth_u16_f16:
10773 case NEON::BI__builtin_neon_vcvtah_s16_f16:
10774 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
10775 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
10776 case NEON::BI__builtin_neon_vcvtph_s16_f16:
10777 case NEON::BI__builtin_neon_vcvth_s16_f16: {
10780 llvm::Type* FTy =
HalfTy;
10781 llvm::Type *Tys[2] = {InTy, FTy};
10783 switch (BuiltinID) {
10784 default: llvm_unreachable(
"missing builtin ID in switch!");
10785 case NEON::BI__builtin_neon_vcvtah_u16_f16:
10786 Int = Intrinsic::aarch64_neon_fcvtau;
break;
10787 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
10788 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
10789 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
10790 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
10791 case NEON::BI__builtin_neon_vcvtph_u16_f16:
10792 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
10793 case NEON::BI__builtin_neon_vcvth_u16_f16:
10794 Int = Intrinsic::aarch64_neon_fcvtzu;
break;
10795 case NEON::BI__builtin_neon_vcvtah_s16_f16:
10796 Int = Intrinsic::aarch64_neon_fcvtas;
break;
10797 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
10798 Int = Intrinsic::aarch64_neon_fcvtms;
break;
10799 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
10800 Int = Intrinsic::aarch64_neon_fcvtns;
break;
10801 case NEON::BI__builtin_neon_vcvtph_s16_f16:
10802 Int = Intrinsic::aarch64_neon_fcvtps;
break;
10803 case NEON::BI__builtin_neon_vcvth_s16_f16:
10804 Int = Intrinsic::aarch64_neon_fcvtzs;
break;
10807 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10809 case NEON::BI__builtin_neon_vcaleh_f16:
10810 case NEON::BI__builtin_neon_vcalth_f16:
10811 case NEON::BI__builtin_neon_vcageh_f16:
10812 case NEON::BI__builtin_neon_vcagth_f16: {
10815 llvm::Type* FTy =
HalfTy;
10816 llvm::Type *Tys[2] = {InTy, FTy};
10818 switch (BuiltinID) {
10819 default: llvm_unreachable(
"missing builtin ID in switch!");
10820 case NEON::BI__builtin_neon_vcageh_f16:
10821 Int = Intrinsic::aarch64_neon_facge;
break;
10822 case NEON::BI__builtin_neon_vcagth_f16:
10823 Int = Intrinsic::aarch64_neon_facgt;
break;
10824 case NEON::BI__builtin_neon_vcaleh_f16:
10825 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
10826 case NEON::BI__builtin_neon_vcalth_f16:
10827 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
10830 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10832 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
10833 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
10836 llvm::Type* FTy =
HalfTy;
10837 llvm::Type *Tys[2] = {InTy, FTy};
10839 switch (BuiltinID) {
10840 default: llvm_unreachable(
"missing builtin ID in switch!");
10841 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
10842 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
10843 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
10844 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
10847 return Builder.CreateTrunc(Ops[0],
Int16Ty);
10849 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
10850 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
10852 llvm::Type* FTy =
HalfTy;
10854 llvm::Type *Tys[2] = {FTy, InTy};
10856 switch (BuiltinID) {
10857 default: llvm_unreachable(
"missing builtin ID in switch!");
10858 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
10859 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
10860 Ops[0] = Builder.CreateSExt(Ops[0], InTy,
"sext");
10862 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
10863 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
10864 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
10869 case NEON::BI__builtin_neon_vpaddd_s64: {
10870 auto *Ty = llvm::FixedVectorType::get(
Int64Ty, 2);
10873 Vec = Builder.CreateBitCast(Vec, Ty,
"v2i64");
10874 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10875 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10876 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10877 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10879 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
10881 case NEON::BI__builtin_neon_vpaddd_f64: {
10882 auto *Ty = llvm::FixedVectorType::get(
DoubleTy, 2);
10885 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f64");
10886 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10887 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10888 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10889 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10891 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
10893 case NEON::BI__builtin_neon_vpadds_f32: {
10894 auto *Ty = llvm::FixedVectorType::get(
FloatTy, 2);
10897 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f32");
10898 llvm::Value *Idx0 = llvm::ConstantInt::get(
SizeTy, 0);
10899 llvm::Value *Idx1 = llvm::ConstantInt::get(
SizeTy, 1);
10900 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
10901 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
10903 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
10905 case NEON::BI__builtin_neon_vceqzd_s64:
10906 case NEON::BI__builtin_neon_vceqzd_f64:
10907 case NEON::BI__builtin_neon_vceqzs_f32:
10908 case NEON::BI__builtin_neon_vceqzh_f16:
10912 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
10913 case NEON::BI__builtin_neon_vcgezd_s64:
10914 case NEON::BI__builtin_neon_vcgezd_f64:
10915 case NEON::BI__builtin_neon_vcgezs_f32:
10916 case NEON::BI__builtin_neon_vcgezh_f16:
10920 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
10921 case NEON::BI__builtin_neon_vclezd_s64:
10922 case NEON::BI__builtin_neon_vclezd_f64:
10923 case NEON::BI__builtin_neon_vclezs_f32:
10924 case NEON::BI__builtin_neon_vclezh_f16:
10928 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
10929 case NEON::BI__builtin_neon_vcgtzd_s64:
10930 case NEON::BI__builtin_neon_vcgtzd_f64:
10931 case NEON::BI__builtin_neon_vcgtzs_f32:
10932 case NEON::BI__builtin_neon_vcgtzh_f16:
10936 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
10937 case NEON::BI__builtin_neon_vcltzd_s64:
10938 case NEON::BI__builtin_neon_vcltzd_f64:
10939 case NEON::BI__builtin_neon_vcltzs_f32:
10940 case NEON::BI__builtin_neon_vcltzh_f16:
10944 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
10946 case NEON::BI__builtin_neon_vceqzd_u64: {
10948 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
10950 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(
Int64Ty));
10951 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vceqzd");
10953 case NEON::BI__builtin_neon_vceqd_f64:
10954 case NEON::BI__builtin_neon_vcled_f64:
10955 case NEON::BI__builtin_neon_vcltd_f64:
10956 case NEON::BI__builtin_neon_vcged_f64:
10957 case NEON::BI__builtin_neon_vcgtd_f64: {
10958 llvm::CmpInst::Predicate
P;
10959 switch (BuiltinID) {
10960 default: llvm_unreachable(
"missing builtin ID in switch!");
10961 case NEON::BI__builtin_neon_vceqd_f64:
P = llvm::FCmpInst::FCMP_OEQ;
break;
10962 case NEON::BI__builtin_neon_vcled_f64:
P = llvm::FCmpInst::FCMP_OLE;
break;
10963 case NEON::BI__builtin_neon_vcltd_f64:
P = llvm::FCmpInst::FCMP_OLT;
break;
10964 case NEON::BI__builtin_neon_vcged_f64:
P = llvm::FCmpInst::FCMP_OGE;
break;
10965 case NEON::BI__builtin_neon_vcgtd_f64:
P = llvm::FCmpInst::FCMP_OGT;
break;
10968 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
10969 Ops[1] = Builder.CreateBitCast(Ops[1],
DoubleTy);
10970 if (
P == llvm::FCmpInst::FCMP_OEQ)
10971 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
10973 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
10974 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vcmpd");
10976 case NEON::BI__builtin_neon_vceqs_f32:
10977 case NEON::BI__builtin_neon_vcles_f32:
10978 case NEON::BI__builtin_neon_vclts_f32:
10979 case NEON::BI__builtin_neon_vcges_f32:
10980 case NEON::BI__builtin_neon_vcgts_f32: {
10981 llvm::CmpInst::Predicate
P;
10982 switch (BuiltinID) {
10983 default: llvm_unreachable(
"missing builtin ID in switch!");
10984 case NEON::BI__builtin_neon_vceqs_f32:
P = llvm::FCmpInst::FCMP_OEQ;
break;
10985 case NEON::BI__builtin_neon_vcles_f32:
P = llvm::FCmpInst::FCMP_OLE;
break;
10986 case NEON::BI__builtin_neon_vclts_f32:
P = llvm::FCmpInst::FCMP_OLT;
break;
10987 case NEON::BI__builtin_neon_vcges_f32:
P = llvm::FCmpInst::FCMP_OGE;
break;
10988 case NEON::BI__builtin_neon_vcgts_f32:
P = llvm::FCmpInst::FCMP_OGT;
break;
10991 Ops[0] = Builder.CreateBitCast(Ops[0],
FloatTy);
10992 Ops[1] = Builder.CreateBitCast(Ops[1],
FloatTy);
10993 if (
P == llvm::FCmpInst::FCMP_OEQ)
10994 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
10996 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
10997 return Builder.CreateSExt(Ops[0],
Int32Ty,
"vcmpd");
10999 case NEON::BI__builtin_neon_vceqh_f16:
11000 case NEON::BI__builtin_neon_vcleh_f16:
11001 case NEON::BI__builtin_neon_vclth_f16:
11002 case NEON::BI__builtin_neon_vcgeh_f16:
11003 case NEON::BI__builtin_neon_vcgth_f16: {
11004 llvm::CmpInst::Predicate
P;
11005 switch (BuiltinID) {
11006 default: llvm_unreachable(
"missing builtin ID in switch!");
11007 case NEON::BI__builtin_neon_vceqh_f16:
P = llvm::FCmpInst::FCMP_OEQ;
break;
11008 case NEON::BI__builtin_neon_vcleh_f16:
P = llvm::FCmpInst::FCMP_OLE;
break;
11009 case NEON::BI__builtin_neon_vclth_f16:
P = llvm::FCmpInst::FCMP_OLT;
break;
11010 case NEON::BI__builtin_neon_vcgeh_f16:
P = llvm::FCmpInst::FCMP_OGE;
break;
11011 case NEON::BI__builtin_neon_vcgth_f16:
P = llvm::FCmpInst::FCMP_OGT;
break;
11014 Ops[0] = Builder.CreateBitCast(Ops[0],
HalfTy);
11015 Ops[1] = Builder.CreateBitCast(Ops[1],
HalfTy);
11016 if (
P == llvm::FCmpInst::FCMP_OEQ)
11017 Ops[0] = Builder.CreateFCmp(
P, Ops[0], Ops[1]);
11019 Ops[0] = Builder.CreateFCmpS(
P, Ops[0], Ops[1]);
11020 return Builder.CreateSExt(Ops[0],
Int16Ty,
"vcmpd");
11022 case NEON::BI__builtin_neon_vceqd_s64:
11023 case NEON::BI__builtin_neon_vceqd_u64:
11024 case NEON::BI__builtin_neon_vcgtd_s64:
11025 case NEON::BI__builtin_neon_vcgtd_u64:
11026 case NEON::BI__builtin_neon_vcltd_s64:
11027 case NEON::BI__builtin_neon_vcltd_u64:
11028 case NEON::BI__builtin_neon_vcged_u64:
11029 case NEON::BI__builtin_neon_vcged_s64:
11030 case NEON::BI__builtin_neon_vcled_u64:
11031 case NEON::BI__builtin_neon_vcled_s64: {
11032 llvm::CmpInst::Predicate
P;
11033 switch (BuiltinID) {
11034 default: llvm_unreachable(
"missing builtin ID in switch!");
11035 case NEON::BI__builtin_neon_vceqd_s64:
11036 case NEON::BI__builtin_neon_vceqd_u64:
P = llvm::ICmpInst::ICMP_EQ;
break;
11037 case NEON::BI__builtin_neon_vcgtd_s64:
P = llvm::ICmpInst::ICMP_SGT;
break;
11038 case NEON::BI__builtin_neon_vcgtd_u64:
P = llvm::ICmpInst::ICMP_UGT;
break;
11039 case NEON::BI__builtin_neon_vcltd_s64:
P = llvm::ICmpInst::ICMP_SLT;
break;
11040 case NEON::BI__builtin_neon_vcltd_u64:
P = llvm::ICmpInst::ICMP_ULT;
break;
11041 case NEON::BI__builtin_neon_vcged_u64:
P = llvm::ICmpInst::ICMP_UGE;
break;
11042 case NEON::BI__builtin_neon_vcged_s64:
P = llvm::ICmpInst::ICMP_SGE;
break;
11043 case NEON::BI__builtin_neon_vcled_u64:
P = llvm::ICmpInst::ICMP_ULE;
break;
11044 case NEON::BI__builtin_neon_vcled_s64:
P = llvm::ICmpInst::ICMP_SLE;
break;
11047 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
11048 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
11049 Ops[0] = Builder.CreateICmp(
P, Ops[0], Ops[1]);
11050 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vceqd");
11052 case NEON::BI__builtin_neon_vtstd_s64:
11053 case NEON::BI__builtin_neon_vtstd_u64: {
11055 Ops[0] = Builder.CreateBitCast(Ops[0],
Int64Ty);
11056 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
11057 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
11058 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
11059 llvm::Constant::getNullValue(
Int64Ty));
11060 return Builder.CreateSExt(Ops[0],
Int64Ty,
"vtstd");
11062 case NEON::BI__builtin_neon_vset_lane_i8:
11063 case NEON::BI__builtin_neon_vset_lane_i16:
11064 case NEON::BI__builtin_neon_vset_lane_i32:
11065 case NEON::BI__builtin_neon_vset_lane_i64:
11066 case NEON::BI__builtin_neon_vset_lane_bf16:
11067 case NEON::BI__builtin_neon_vset_lane_f32:
11068 case NEON::BI__builtin_neon_vsetq_lane_i8:
11069 case NEON::BI__builtin_neon_vsetq_lane_i16:
11070 case NEON::BI__builtin_neon_vsetq_lane_i32:
11071 case NEON::BI__builtin_neon_vsetq_lane_i64:
11072 case NEON::BI__builtin_neon_vsetq_lane_bf16:
11073 case NEON::BI__builtin_neon_vsetq_lane_f32:
11075 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11076 case NEON::BI__builtin_neon_vset_lane_f64:
11079 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 1));
11081 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11082 case NEON::BI__builtin_neon_vsetq_lane_f64:
11085 Builder.CreateBitCast(Ops[1], llvm::FixedVectorType::get(
DoubleTy, 2));
11087 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
11089 case NEON::BI__builtin_neon_vget_lane_i8:
11090 case NEON::BI__builtin_neon_vdupb_lane_i8:
11092 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 8));
11095 case NEON::BI__builtin_neon_vgetq_lane_i8:
11096 case NEON::BI__builtin_neon_vdupb_laneq_i8:
11098 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int8Ty, 16));
11101 case NEON::BI__builtin_neon_vget_lane_i16:
11102 case NEON::BI__builtin_neon_vduph_lane_i16:
11104 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 4));
11107 case NEON::BI__builtin_neon_vgetq_lane_i16:
11108 case NEON::BI__builtin_neon_vduph_laneq_i16:
11110 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int16Ty, 8));
11113 case NEON::BI__builtin_neon_vget_lane_i32:
11114 case NEON::BI__builtin_neon_vdups_lane_i32:
11116 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 2));
11119 case NEON::BI__builtin_neon_vdups_lane_f32:
11121 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11124 case NEON::BI__builtin_neon_vgetq_lane_i32:
11125 case NEON::BI__builtin_neon_vdups_laneq_i32:
11127 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int32Ty, 4));
11130 case NEON::BI__builtin_neon_vget_lane_i64:
11131 case NEON::BI__builtin_neon_vdupd_lane_i64:
11133 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 1));
11136 case NEON::BI__builtin_neon_vdupd_lane_f64:
11138 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11141 case NEON::BI__builtin_neon_vgetq_lane_i64:
11142 case NEON::BI__builtin_neon_vdupd_laneq_i64:
11144 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
Int64Ty, 2));
11147 case NEON::BI__builtin_neon_vget_lane_f32:
11149 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 2));
11152 case NEON::BI__builtin_neon_vget_lane_f64:
11154 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 1));
11157 case NEON::BI__builtin_neon_vgetq_lane_f32:
11158 case NEON::BI__builtin_neon_vdups_laneq_f32:
11160 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
FloatTy, 4));
11163 case NEON::BI__builtin_neon_vgetq_lane_f64:
11164 case NEON::BI__builtin_neon_vdupd_laneq_f64:
11166 Builder.CreateBitCast(Ops[0], llvm::FixedVectorType::get(
DoubleTy, 2));
11169 case NEON::BI__builtin_neon_vaddh_f16:
11171 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
11172 case NEON::BI__builtin_neon_vsubh_f16:
11174 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
11175 case NEON::BI__builtin_neon_vmulh_f16:
11177 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
11178 case NEON::BI__builtin_neon_vdivh_f16:
11180 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
11181 case NEON::BI__builtin_neon_vfmah_f16:
11184 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
11186 case NEON::BI__builtin_neon_vfmsh_f16: {
11191 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
HalfTy,
11194 case NEON::BI__builtin_neon_vaddd_s64:
11195 case NEON::BI__builtin_neon_vaddd_u64:
11197 case NEON::BI__builtin_neon_vsubd_s64:
11198 case NEON::BI__builtin_neon_vsubd_u64:
11200 case NEON::BI__builtin_neon_vqdmlalh_s16:
11201 case NEON::BI__builtin_neon_vqdmlslh_s16: {
11205 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
11207 ProductOps,
"vqdmlXl");
11208 Constant *CI = ConstantInt::get(
SizeTy, 0);
11209 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
11211 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
11212 ? Intrinsic::aarch64_neon_sqadd
11213 : Intrinsic::aarch64_neon_sqsub;
11216 case NEON::BI__builtin_neon_vqshlud_n_s64: {
11218 Ops[1] = Builder.CreateZExt(Ops[1],
Int64Ty);
11222 case NEON::BI__builtin_neon_vqshld_n_u64:
11223 case NEON::BI__builtin_neon_vqshld_n_s64: {
11224 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
11225 ? Intrinsic::aarch64_neon_uqshl
11226 : Intrinsic::aarch64_neon_sqshl;
11228 Ops[1] = Builder.CreateZExt(Ops[1],
Int64Ty);
11231 case NEON::BI__builtin_neon_vrshrd_n_u64:
11232 case NEON::BI__builtin_neon_vrshrd_n_s64: {
11233 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
11234 ? Intrinsic::aarch64_neon_urshl
11235 : Intrinsic::aarch64_neon_srshl;
11238 Ops[1] = ConstantInt::get(
Int64Ty, -SV);
11241 case NEON::BI__builtin_neon_vrsrad_n_u64:
11242 case NEON::BI__builtin_neon_vrsrad_n_s64: {
11243 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
11244 ? Intrinsic::aarch64_neon_urshl
11245 : Intrinsic::aarch64_neon_srshl;
11246 Ops[1] = Builder.CreateBitCast(Ops[1],
Int64Ty);
11249 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
11250 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1],
Int64Ty));
11252 case NEON::BI__builtin_neon_vshld_n_s64:
11253 case NEON::BI__builtin_neon_vshld_n_u64: {
11255 return Builder.CreateShl(
11256 Ops[0], ConstantInt::get(
Int64Ty, Amt->getZExtValue()),
"shld_n");
11258 case NEON::BI__builtin_neon_vshrd_n_s64: {
11260 return Builder.CreateAShr(
11261 Ops[0], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t
>(63),
11262 Amt->getZExtValue())),
11265 case NEON::BI__builtin_neon_vshrd_n_u64: {
11267 uint64_t ShiftAmt = Amt->getZExtValue();
11269 if (ShiftAmt == 64)
11270 return ConstantInt::get(
Int64Ty, 0);
11271 return Builder.CreateLShr(Ops[0], ConstantInt::get(
Int64Ty, ShiftAmt),
11274 case NEON::BI__builtin_neon_vsrad_n_s64: {
11276 Ops[1] = Builder.CreateAShr(
11277 Ops[1], ConstantInt::get(
Int64Ty, std::min(
static_cast<uint64_t
>(63),
11278 Amt->getZExtValue())),
11280 return Builder.CreateAdd(Ops[0], Ops[1]);
11282 case NEON::BI__builtin_neon_vsrad_n_u64: {
11284 uint64_t ShiftAmt = Amt->getZExtValue();
11287 if (ShiftAmt == 64)
11289 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, ShiftAmt),
11291 return Builder.CreateAdd(Ops[0], Ops[1]);
11293 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
11294 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
11295 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
11296 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
11302 auto *VTy = llvm::FixedVectorType::get(
Int32Ty, 4);
11304 ProductOps,
"vqdmlXl");
11305 Constant *CI = ConstantInt::get(
SizeTy, 0);
11306 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
11309 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
11310 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
11311 ? Intrinsic::aarch64_neon_sqadd
11312 : Intrinsic::aarch64_neon_sqsub;
11315 case NEON::BI__builtin_neon_vqdmlals_s32:
11316 case NEON::BI__builtin_neon_vqdmlsls_s32: {
11318 ProductOps.push_back(Ops[1]);
11322 ProductOps,
"vqdmlXl");
11324 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
11325 ? Intrinsic::aarch64_neon_sqadd
11326 : Intrinsic::aarch64_neon_sqsub;
11329 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
11330 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
11331 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
11332 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
11336 ProductOps.push_back(Ops[1]);
11337 ProductOps.push_back(Ops[2]);
11340 ProductOps,
"vqdmlXl");
11343 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
11344 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
11345 ? Intrinsic::aarch64_neon_sqadd
11346 : Intrinsic::aarch64_neon_sqsub;
11349 case NEON::BI__builtin_neon_vget_lane_bf16:
11350 case NEON::BI__builtin_neon_vduph_lane_bf16:
11351 case NEON::BI__builtin_neon_vduph_lane_f16: {
11355 case NEON::BI__builtin_neon_vgetq_lane_bf16:
11356 case NEON::BI__builtin_neon_vduph_laneq_bf16:
11357 case NEON::BI__builtin_neon_vduph_laneq_f16: {
11362 case clang::AArch64::BI_InterlockedAdd: {
11365 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
11366 AtomicRMWInst::Add, Arg0, Arg1,
11367 llvm::AtomicOrdering::SequentiallyConsistent);
11368 return Builder.CreateAdd(RMWI, Arg1);
11373 llvm::Type *Ty = VTy;
11384 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
11385 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
11392 switch (BuiltinID) {
11393 default:
return nullptr;
11394 case NEON::BI__builtin_neon_vbsl_v:
11395 case NEON::BI__builtin_neon_vbslq_v: {
11396 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
11397 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
11398 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
11399 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
11401 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
11402 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
11403 Ops[0] = Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
11404 return Builder.CreateBitCast(Ops[0], Ty);
11406 case NEON::BI__builtin_neon_vfma_lane_v:
11407 case NEON::BI__builtin_neon_vfmaq_lane_v: {
11410 Value *Addend = Ops[0];
11411 Value *Multiplicand = Ops[1];
11412 Value *LaneSource = Ops[2];
11413 Ops[0] = Multiplicand;
11414 Ops[1] = LaneSource;
11418 auto *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v
11419 ? llvm::FixedVectorType::get(VTy->getElementType(),
11420 VTy->getNumElements() / 2)
11423 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(), cst);
11424 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
11425 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
11428 Int = Builder.getIsFPConstrained() ? Intrinsic::experimental_constrained_fma
11432 case NEON::BI__builtin_neon_vfma_laneq_v: {
11435 if (VTy && VTy->getElementType() ==
DoubleTy) {
11436 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
11437 Ops[1] = Builder.CreateBitCast(Ops[1],
DoubleTy);
11438 llvm::FixedVectorType *VTy =
11440 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
11441 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
11444 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma,
11445 DoubleTy, {Ops[1], Ops[2], Ops[0]});
11446 return Builder.CreateBitCast(
Result, Ty);
11448 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11449 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11451 auto *STy = llvm::FixedVectorType::get(VTy->getElementType(),
11452 VTy->getNumElements() * 2);
11453 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
11454 Value *SV = llvm::ConstantVector::getSplat(VTy->getElementCount(),
11455 cast<ConstantInt>(Ops[3]));
11456 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
11459 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
11460 {Ops[2], Ops[1], Ops[0]});
11462 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
11463 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11464 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
11466 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
11469 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
11470 {Ops[2], Ops[1], Ops[0]});
11472 case NEON::BI__builtin_neon_vfmah_lane_f16:
11473 case NEON::BI__builtin_neon_vfmas_lane_f32:
11474 case NEON::BI__builtin_neon_vfmah_laneq_f16:
11475 case NEON::BI__builtin_neon_vfmas_laneq_f32:
11476 case NEON::BI__builtin_neon_vfmad_lane_f64:
11477 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
11480 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
11482 *
this, Intrinsic::fma, Intrinsic::experimental_constrained_fma, Ty,
11483 {Ops[1], Ops[2], Ops[0]});
11485 case NEON::BI__builtin_neon_vmull_v:
11487 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
11488 if (
Type.isPoly()) Int = Intrinsic::aarch64_neon_pmull;
11490 case NEON::BI__builtin_neon_vmax_v:
11491 case NEON::BI__builtin_neon_vmaxq_v:
11493 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
11494 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
11496 case NEON::BI__builtin_neon_vmaxh_f16: {
11498 Int = Intrinsic::aarch64_neon_fmax;
11501 case NEON::BI__builtin_neon_vmin_v:
11502 case NEON::BI__builtin_neon_vminq_v:
11504 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
11505 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
11507 case NEON::BI__builtin_neon_vminh_f16: {
11509 Int = Intrinsic::aarch64_neon_fmin;
11512 case NEON::BI__builtin_neon_vabd_v:
11513 case NEON::BI__builtin_neon_vabdq_v:
11515 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
11516 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
11518 case NEON::BI__builtin_neon_vpadal_v:
11519 case NEON::BI__builtin_neon_vpadalq_v: {
11520 unsigned ArgElts = VTy->getNumElements();
11522 unsigned BitWidth = EltTy->getBitWidth();
11523 auto *ArgTy = llvm::FixedVectorType::get(
11524 llvm::IntegerType::get(
getLLVMContext(), BitWidth / 2), 2 * ArgElts);
11525 llvm::Type* Tys[2] = { VTy, ArgTy };
11526 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
11528 TmpOps.push_back(Ops[1]);
11531 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
11532 return Builder.CreateAdd(tmp, addend);
11534 case NEON::BI__builtin_neon_vpmin_v:
11535 case NEON::BI__builtin_neon_vpminq_v:
11537 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
11538 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
11540 case NEON::BI__builtin_neon_vpmax_v:
11541 case NEON::BI__builtin_neon_vpmaxq_v:
11543 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
11544 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
11546 case NEON::BI__builtin_neon_vminnm_v:
11547 case NEON::BI__builtin_neon_vminnmq_v:
11548 Int = Intrinsic::aarch64_neon_fminnm;
11550 case NEON::BI__builtin_neon_vminnmh_f16:
11552 Int = Intrinsic::aarch64_neon_fminnm;
11554 case NEON::BI__builtin_neon_vmaxnm_v:
11555 case NEON::BI__builtin_neon_vmaxnmq_v:
11556 Int = Intrinsic::aarch64_neon_fmaxnm;
11558 case NEON::BI__builtin_neon_vmaxnmh_f16:
11560 Int = Intrinsic::aarch64_neon_fmaxnm;
11562 case NEON::BI__builtin_neon_vrecpss_f32: {
11567 case NEON::BI__builtin_neon_vrecpsd_f64:
11571 case NEON::BI__builtin_neon_vrecpsh_f16:
11575 case NEON::BI__builtin_neon_vqshrun_n_v:
11576 Int = Intrinsic::aarch64_neon_sqshrun;
11578 case NEON::BI__builtin_neon_vqrshrun_n_v:
11579 Int = Intrinsic::aarch64_neon_sqrshrun;
11581 case NEON::BI__builtin_neon_vqshrn_n_v:
11582 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
11584 case NEON::BI__builtin_neon_vrshrn_n_v:
11585 Int = Intrinsic::aarch64_neon_rshrn;
11587 case NEON::BI__builtin_neon_vqrshrn_n_v:
11588 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
11590 case NEON::BI__builtin_neon_vrndah_f16: {
11592 Int = Builder.getIsFPConstrained()
11593 ? Intrinsic::experimental_constrained_round
11594 : Intrinsic::round;
11597 case NEON::BI__builtin_neon_vrnda_v:
11598 case NEON::BI__builtin_neon_vrndaq_v: {
11599 Int = Builder.getIsFPConstrained()
11600 ? Intrinsic::experimental_constrained_round
11601 : Intrinsic::round;
11604 case NEON::BI__builtin_neon_vrndih_f16: {
11606 Int = Builder.getIsFPConstrained()
11607 ? Intrinsic::experimental_constrained_nearbyint
11608 : Intrinsic::nearbyint;
11611 case NEON::BI__builtin_neon_vrndmh_f16: {
11613 Int = Builder.getIsFPConstrained()
11614 ? Intrinsic::experimental_constrained_floor
11615 : Intrinsic::floor;
11618 case NEON::BI__builtin_neon_vrndm_v:
11619 case NEON::BI__builtin_neon_vrndmq_v: {
11620 Int = Builder.getIsFPConstrained()
11621 ? Intrinsic::experimental_constrained_floor
11622 : Intrinsic::floor;
11625 case NEON::BI__builtin_neon_vrndnh_f16: {
11627 Int = Builder.getIsFPConstrained()
11628 ? Intrinsic::experimental_constrained_roundeven
11629 : Intrinsic::roundeven;
11632 case NEON::BI__builtin_neon_vrndn_v:
11633 case NEON::BI__builtin_neon_vrndnq_v: {
11634 Int = Builder.getIsFPConstrained()
11635 ? Intrinsic::experimental_constrained_roundeven
11636 : Intrinsic::roundeven;
11639 case NEON::BI__builtin_neon_vrndns_f32: {
11641 Int = Builder.getIsFPConstrained()
11642 ? Intrinsic::experimental_constrained_roundeven
11643 : Intrinsic::roundeven;
11646 case NEON::BI__builtin_neon_vrndph_f16: {
11648 Int = Builder.getIsFPConstrained()
11649 ? Intrinsic::experimental_constrained_ceil
11653 case NEON::BI__builtin_neon_vrndp_v:
11654 case NEON::BI__builtin_neon_vrndpq_v: {
11655 Int = Builder.getIsFPConstrained()
11656 ? Intrinsic::experimental_constrained_ceil
11660 case NEON::BI__builtin_neon_vrndxh_f16: {
11662 Int = Builder.getIsFPConstrained()
11663 ? Intrinsic::experimental_constrained_rint
11667 case NEON::BI__builtin_neon_vrndx_v:
11668 case NEON::BI__builtin_neon_vrndxq_v: {
11669 Int = Builder.getIsFPConstrained()
11670 ? Intrinsic::experimental_constrained_rint
11674 case NEON::BI__builtin_neon_vrndh_f16: {
11676 Int = Builder.getIsFPConstrained()
11677 ? Intrinsic::experimental_constrained_trunc
11678 : Intrinsic::trunc;
11681 case NEON::BI__builtin_neon_vrnd32x_f32:
11682 case NEON::BI__builtin_neon_vrnd32xq_f32: {
11684 Int = Intrinsic::aarch64_neon_frint32x;
11687 case NEON::BI__builtin_neon_vrnd32z_f32:
11688 case NEON::BI__builtin_neon_vrnd32zq_f32: {
11690 Int = Intrinsic::aarch64_neon_frint32z;
11693 case NEON::BI__builtin_neon_vrnd64x_f32:
11694 case NEON::BI__builtin_neon_vrnd64xq_f32: {
11696 Int = Intrinsic::aarch64_neon_frint64x;
11699 case NEON::BI__builtin_neon_vrnd64z_f32:
11700 case NEON::BI__builtin_neon_vrnd64zq_f32: {
11702 Int = Intrinsic::aarch64_neon_frint64z;
11705 case NEON::BI__builtin_neon_vrnd_v:
11706 case NEON::BI__builtin_neon_vrndq_v: {
11707 Int = Builder.getIsFPConstrained()
11708 ? Intrinsic::experimental_constrained_trunc
11709 : Intrinsic::trunc;
11712 case NEON::BI__builtin_neon_vcvt_f64_v:
11713 case NEON::BI__builtin_neon_vcvtq_f64_v:
11714 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11716 return usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
11717 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
11718 case NEON::BI__builtin_neon_vcvt_f64_f32: {
11720 "unexpected vcvt_f64_f32 builtin");
11722 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
11724 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
11726 case NEON::BI__builtin_neon_vcvt_f32_f64: {
11728 "unexpected vcvt_f32_f64 builtin");
11730 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
11732 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
11734 case NEON::BI__builtin_neon_vcvt_s32_v:
11735 case NEON::BI__builtin_neon_vcvt_u32_v:
11736 case NEON::BI__builtin_neon_vcvt_s64_v:
11737 case NEON::BI__builtin_neon_vcvt_u64_v:
11738 case NEON::BI__builtin_neon_vcvt_s16_f16:
11739 case NEON::BI__builtin_neon_vcvt_u16_f16:
11740 case NEON::BI__builtin_neon_vcvtq_s32_v:
11741 case NEON::BI__builtin_neon_vcvtq_u32_v:
11742 case NEON::BI__builtin_neon_vcvtq_s64_v:
11743 case NEON::BI__builtin_neon_vcvtq_u64_v:
11744 case NEON::BI__builtin_neon_vcvtq_s16_f16:
11745 case NEON::BI__builtin_neon_vcvtq_u16_f16: {
11747 usgn ? Intrinsic::aarch64_neon_fcvtzu : Intrinsic::aarch64_neon_fcvtzs;
11751 case NEON::BI__builtin_neon_vcvta_s16_f16:
11752 case NEON::BI__builtin_neon_vcvta_u16_f16:
11753 case NEON::BI__builtin_neon_vcvta_s32_v:
11754 case NEON::BI__builtin_neon_vcvtaq_s16_f16:
11755 case NEON::BI__builtin_neon_vcvtaq_s32_v:
11756 case NEON::BI__builtin_neon_vcvta_u32_v:
11757 case NEON::BI__builtin_neon_vcvtaq_u16_f16:
11758 case NEON::BI__builtin_neon_vcvtaq_u32_v:
11759 case NEON::BI__builtin_neon_vcvta_s64_v:
11760 case NEON::BI__builtin_neon_vcvtaq_s64_v:
11761 case NEON::BI__builtin_neon_vcvta_u64_v:
11762 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
11763 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
11767 case NEON::BI__builtin_neon_vcvtm_s16_f16:
11768 case NEON::BI__builtin_neon_vcvtm_s32_v:
11769 case NEON::BI__builtin_neon_vcvtmq_s16_f16:
11770 case NEON::BI__builtin_neon_vcvtmq_s32_v:
11771 case NEON::BI__builtin_neon_vcvtm_u16_f16:
11772 case NEON::BI__builtin_neon_vcvtm_u32_v:
11773 case NEON::BI__builtin_neon_vcvtmq_u16_f16:
11774 case NEON::BI__builtin_neon_vcvtmq_u32_v:
11775 case NEON::BI__builtin_neon_vcvtm_s64_v:
11776 case NEON::BI__builtin_neon_vcvtmq_s64_v:
11777 case NEON::BI__builtin_neon_vcvtm_u64_v:
11778 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
11779 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
11783 case NEON::BI__builtin_neon_vcvtn_s16_f16:
11784 case NEON::BI__builtin_neon_vcvtn_s32_v:
11785 case NEON::BI__builtin_neon_vcvtnq_s16_f16:
11786 case NEON::BI__builtin_neon_vcvtnq_s32_v:
11787 case NEON::BI__builtin_neon_vcvtn_u16_f16:
11788 case NEON::BI__builtin_neon_vcvtn_u32_v:
11789 case NEON::BI__builtin_neon_vcvtnq_u16_f16:
11790 case NEON::BI__builtin_neon_vcvtnq_u32_v:
11791 case NEON::BI__builtin_neon_vcvtn_s64_v:
11792 case NEON::BI__builtin_neon_vcvtnq_s64_v:
11793 case NEON::BI__builtin_neon_vcvtn_u64_v:
11794 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
11795 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
11799 case NEON::BI__builtin_neon_vcvtp_s16_f16:
11800 case NEON::BI__builtin_neon_vcvtp_s32_v:
11801 case NEON::BI__builtin_neon_vcvtpq_s16_f16:
11802 case NEON::BI__builtin_neon_vcvtpq_s32_v:
11803 case NEON::BI__builtin_neon_vcvtp_u16_f16:
11804 case NEON::BI__builtin_neon_vcvtp_u32_v:
11805 case NEON::BI__builtin_neon_vcvtpq_u16_f16:
11806 case NEON::BI__builtin_neon_vcvtpq_u32_v:
11807 case NEON::BI__builtin_neon_vcvtp_s64_v:
11808 case NEON::BI__builtin_neon_vcvtpq_s64_v:
11809 case NEON::BI__builtin_neon_vcvtp_u64_v:
11810 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
11811 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
11815 case NEON::BI__builtin_neon_vmulx_v:
11816 case NEON::BI__builtin_neon_vmulxq_v: {
11817 Int = Intrinsic::aarch64_neon_fmulx;
11820 case NEON::BI__builtin_neon_vmulxh_lane_f16:
11821 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
11825 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
11827 Int = Intrinsic::aarch64_neon_fmulx;
11830 case NEON::BI__builtin_neon_vmul_lane_v:
11831 case NEON::BI__builtin_neon_vmul_laneq_v: {
11834 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
11836 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
11837 llvm::FixedVectorType *VTy =
11839 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
11840 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
11841 Value *
Result = Builder.CreateFMul(Ops[0], Ops[1]);
11842 return Builder.CreateBitCast(
Result, Ty);
11844 case NEON::BI__builtin_neon_vnegd_s64:
11846 case NEON::BI__builtin_neon_vnegh_f16:
11848 case NEON::BI__builtin_neon_vpmaxnm_v:
11849 case NEON::BI__builtin_neon_vpmaxnmq_v: {
11850 Int = Intrinsic::aarch64_neon_fmaxnmp;
11853 case NEON::BI__builtin_neon_vpminnm_v:
11854 case NEON::BI__builtin_neon_vpminnmq_v: {
11855 Int = Intrinsic::aarch64_neon_fminnmp;
11858 case NEON::BI__builtin_neon_vsqrth_f16: {
11860 Int = Builder.getIsFPConstrained()
11861 ? Intrinsic::experimental_constrained_sqrt
11865 case NEON::BI__builtin_neon_vsqrt_v:
11866 case NEON::BI__builtin_neon_vsqrtq_v: {
11867 Int = Builder.getIsFPConstrained()
11868 ? Intrinsic::experimental_constrained_sqrt
11870 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
11873 case NEON::BI__builtin_neon_vrbit_v:
11874 case NEON::BI__builtin_neon_vrbitq_v: {
11875 Int = Intrinsic::bitreverse;
11878 case NEON::BI__builtin_neon_vaddv_u8:
11882 case NEON::BI__builtin_neon_vaddv_s8: {
11883 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11885 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11886 llvm::Type *Tys[2] = { Ty, VTy };
11889 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11891 case NEON::BI__builtin_neon_vaddv_u16:
11894 case NEON::BI__builtin_neon_vaddv_s16: {
11895 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11897 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11898 llvm::Type *Tys[2] = { Ty, VTy };
11901 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11903 case NEON::BI__builtin_neon_vaddvq_u8:
11906 case NEON::BI__builtin_neon_vaddvq_s8: {
11907 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11909 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11910 llvm::Type *Tys[2] = { Ty, VTy };
11913 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11915 case NEON::BI__builtin_neon_vaddvq_u16:
11918 case NEON::BI__builtin_neon_vaddvq_s16: {
11919 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
11921 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11922 llvm::Type *Tys[2] = { Ty, VTy };
11925 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11927 case NEON::BI__builtin_neon_vmaxv_u8: {
11928 Int = Intrinsic::aarch64_neon_umaxv;
11930 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11931 llvm::Type *Tys[2] = { Ty, VTy };
11934 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11936 case NEON::BI__builtin_neon_vmaxv_u16: {
11937 Int = Intrinsic::aarch64_neon_umaxv;
11939 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11940 llvm::Type *Tys[2] = { Ty, VTy };
11943 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11945 case NEON::BI__builtin_neon_vmaxvq_u8: {
11946 Int = Intrinsic::aarch64_neon_umaxv;
11948 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11949 llvm::Type *Tys[2] = { Ty, VTy };
11952 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11954 case NEON::BI__builtin_neon_vmaxvq_u16: {
11955 Int = Intrinsic::aarch64_neon_umaxv;
11957 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11958 llvm::Type *Tys[2] = { Ty, VTy };
11961 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11963 case NEON::BI__builtin_neon_vmaxv_s8: {
11964 Int = Intrinsic::aarch64_neon_smaxv;
11966 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
11967 llvm::Type *Tys[2] = { Ty, VTy };
11970 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11972 case NEON::BI__builtin_neon_vmaxv_s16: {
11973 Int = Intrinsic::aarch64_neon_smaxv;
11975 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
11976 llvm::Type *Tys[2] = { Ty, VTy };
11979 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11981 case NEON::BI__builtin_neon_vmaxvq_s8: {
11982 Int = Intrinsic::aarch64_neon_smaxv;
11984 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
11985 llvm::Type *Tys[2] = { Ty, VTy };
11988 return Builder.CreateTrunc(Ops[0],
Int8Ty);
11990 case NEON::BI__builtin_neon_vmaxvq_s16: {
11991 Int = Intrinsic::aarch64_neon_smaxv;
11993 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
11994 llvm::Type *Tys[2] = { Ty, VTy };
11997 return Builder.CreateTrunc(Ops[0],
Int16Ty);
11999 case NEON::BI__builtin_neon_vmaxv_f16: {
12000 Int = Intrinsic::aarch64_neon_fmaxv;
12002 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12003 llvm::Type *Tys[2] = { Ty, VTy };
12006 return Builder.CreateTrunc(Ops[0],
HalfTy);
12008 case NEON::BI__builtin_neon_vmaxvq_f16: {
12009 Int = Intrinsic::aarch64_neon_fmaxv;
12011 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12012 llvm::Type *Tys[2] = { Ty, VTy };
12015 return Builder.CreateTrunc(Ops[0],
HalfTy);
12017 case NEON::BI__builtin_neon_vminv_u8: {
12018 Int = Intrinsic::aarch64_neon_uminv;
12020 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12021 llvm::Type *Tys[2] = { Ty, VTy };
12024 return Builder.CreateTrunc(Ops[0],
Int8Ty);
12026 case NEON::BI__builtin_neon_vminv_u16: {
12027 Int = Intrinsic::aarch64_neon_uminv;
12029 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12030 llvm::Type *Tys[2] = { Ty, VTy };
12033 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12035 case NEON::BI__builtin_neon_vminvq_u8: {
12036 Int = Intrinsic::aarch64_neon_uminv;
12038 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12039 llvm::Type *Tys[2] = { Ty, VTy };
12042 return Builder.CreateTrunc(Ops[0],
Int8Ty);
12044 case NEON::BI__builtin_neon_vminvq_u16: {
12045 Int = Intrinsic::aarch64_neon_uminv;
12047 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12048 llvm::Type *Tys[2] = { Ty, VTy };
12051 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12053 case NEON::BI__builtin_neon_vminv_s8: {
12054 Int = Intrinsic::aarch64_neon_sminv;
12056 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12057 llvm::Type *Tys[2] = { Ty, VTy };
12060 return Builder.CreateTrunc(Ops[0],
Int8Ty);
12062 case NEON::BI__builtin_neon_vminv_s16: {
12063 Int = Intrinsic::aarch64_neon_sminv;
12065 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12066 llvm::Type *Tys[2] = { Ty, VTy };
12069 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12071 case NEON::BI__builtin_neon_vminvq_s8: {
12072 Int = Intrinsic::aarch64_neon_sminv;
12074 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12075 llvm::Type *Tys[2] = { Ty, VTy };
12078 return Builder.CreateTrunc(Ops[0],
Int8Ty);
12080 case NEON::BI__builtin_neon_vminvq_s16: {
12081 Int = Intrinsic::aarch64_neon_sminv;
12083 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12084 llvm::Type *Tys[2] = { Ty, VTy };
12087 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12089 case NEON::BI__builtin_neon_vminv_f16: {
12090 Int = Intrinsic::aarch64_neon_fminv;
12092 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12093 llvm::Type *Tys[2] = { Ty, VTy };
12096 return Builder.CreateTrunc(Ops[0],
HalfTy);
12098 case NEON::BI__builtin_neon_vminvq_f16: {
12099 Int = Intrinsic::aarch64_neon_fminv;
12101 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12102 llvm::Type *Tys[2] = { Ty, VTy };
12105 return Builder.CreateTrunc(Ops[0],
HalfTy);
12107 case NEON::BI__builtin_neon_vmaxnmv_f16: {
12108 Int = Intrinsic::aarch64_neon_fmaxnmv;
12110 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12111 llvm::Type *Tys[2] = { Ty, VTy };
12114 return Builder.CreateTrunc(Ops[0],
HalfTy);
12116 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
12117 Int = Intrinsic::aarch64_neon_fmaxnmv;
12119 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12120 llvm::Type *Tys[2] = { Ty, VTy };
12123 return Builder.CreateTrunc(Ops[0],
HalfTy);
12125 case NEON::BI__builtin_neon_vminnmv_f16: {
12126 Int = Intrinsic::aarch64_neon_fminnmv;
12128 VTy = llvm::FixedVectorType::get(
HalfTy, 4);
12129 llvm::Type *Tys[2] = { Ty, VTy };
12132 return Builder.CreateTrunc(Ops[0],
HalfTy);
12134 case NEON::BI__builtin_neon_vminnmvq_f16: {
12135 Int = Intrinsic::aarch64_neon_fminnmv;
12137 VTy = llvm::FixedVectorType::get(
HalfTy, 8);
12138 llvm::Type *Tys[2] = { Ty, VTy };
12141 return Builder.CreateTrunc(Ops[0],
HalfTy);
12143 case NEON::BI__builtin_neon_vmul_n_f64: {
12144 Ops[0] = Builder.CreateBitCast(Ops[0],
DoubleTy);
12146 return Builder.CreateFMul(Ops[0], RHS);
12148 case NEON::BI__builtin_neon_vaddlv_u8: {
12149 Int = Intrinsic::aarch64_neon_uaddlv;
12151 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12152 llvm::Type *Tys[2] = { Ty, VTy };
12155 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12157 case NEON::BI__builtin_neon_vaddlv_u16: {
12158 Int = Intrinsic::aarch64_neon_uaddlv;
12160 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12161 llvm::Type *Tys[2] = { Ty, VTy };
12165 case NEON::BI__builtin_neon_vaddlvq_u8: {
12166 Int = Intrinsic::aarch64_neon_uaddlv;
12168 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12169 llvm::Type *Tys[2] = { Ty, VTy };
12172 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12174 case NEON::BI__builtin_neon_vaddlvq_u16: {
12175 Int = Intrinsic::aarch64_neon_uaddlv;
12177 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12178 llvm::Type *Tys[2] = { Ty, VTy };
12182 case NEON::BI__builtin_neon_vaddlv_s8: {
12183 Int = Intrinsic::aarch64_neon_saddlv;
12185 VTy = llvm::FixedVectorType::get(
Int8Ty, 8);
12186 llvm::Type *Tys[2] = { Ty, VTy };
12189 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12191 case NEON::BI__builtin_neon_vaddlv_s16: {
12192 Int = Intrinsic::aarch64_neon_saddlv;
12194 VTy = llvm::FixedVectorType::get(
Int16Ty, 4);
12195 llvm::Type *Tys[2] = { Ty, VTy };
12199 case NEON::BI__builtin_neon_vaddlvq_s8: {
12200 Int = Intrinsic::aarch64_neon_saddlv;
12202 VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
12203 llvm::Type *Tys[2] = { Ty, VTy };
12206 return Builder.CreateTrunc(Ops[0],
Int16Ty);
12208 case NEON::BI__builtin_neon_vaddlvq_s16: {
12209 Int = Intrinsic::aarch64_neon_saddlv;
12211 VTy = llvm::FixedVectorType::get(
Int16Ty, 8);
12212 llvm::Type *Tys[2] = { Ty, VTy };
12216 case NEON::BI__builtin_neon_vsri_n_v:
12217 case NEON::BI__builtin_neon_vsriq_n_v: {
12218 Int = Intrinsic::aarch64_neon_vsri;
12222 case NEON::BI__builtin_neon_vsli_n_v:
12223 case NEON::BI__builtin_neon_vsliq_n_v: {
12224 Int = Intrinsic::aarch64_neon_vsli;
12228 case NEON::BI__builtin_neon_vsra_n_v:
12229 case NEON::BI__builtin_neon_vsraq_n_v:
12230 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12232 return Builder.CreateAdd(Ops[0], Ops[1]);
12233 case NEON::BI__builtin_neon_vrsra_n_v:
12234 case NEON::BI__builtin_neon_vrsraq_n_v: {
12235 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
12237 TmpOps.push_back(Ops[1]);
12238 TmpOps.push_back(Ops[2]);
12240 llvm::Value *tmp =
EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
12241 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
12242 return Builder.CreateAdd(Ops[0], tmp);
12244 case NEON::BI__builtin_neon_vld1_v:
12245 case NEON::BI__builtin_neon_vld1q_v: {
12246 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
12247 return Builder.CreateAlignedLoad(VTy, Ops[0], PtrOp0.
getAlignment());
12249 case NEON::BI__builtin_neon_vst1_v:
12250 case NEON::BI__builtin_neon_vst1q_v:
12251 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
12252 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
12253 return Builder.CreateAlignedStore(Ops[1], Ops[0], PtrOp0.
getAlignment());
12254 case NEON::BI__builtin_neon_vld1_lane_v:
12255 case NEON::BI__builtin_neon_vld1q_lane_v: {
12256 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12257 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
12258 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12259 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
12261 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
12263 case NEON::BI__builtin_neon_vldap1_lane_s64:
12264 case NEON::BI__builtin_neon_vldap1q_lane_s64: {
12265 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12266 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
12267 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12268 llvm::LoadInst *LI = Builder.CreateAlignedLoad(
12269 VTy->getElementType(), Ops[0], PtrOp0.
getAlignment());
12270 LI->setAtomic(llvm::AtomicOrdering::Acquire);
12272 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vldap1_lane");
12274 case NEON::BI__builtin_neon_vld1_dup_v:
12275 case NEON::BI__builtin_neon_vld1q_dup_v: {
12276 Value *
V = PoisonValue::get(Ty);
12277 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
12278 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12279 Ops[0] = Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0],
12281 llvm::Constant *CI = ConstantInt::get(
Int32Ty, 0);
12282 Ops[0] = Builder.CreateInsertElement(
V, Ops[0], CI);
12285 case NEON::BI__builtin_neon_vst1_lane_v:
12286 case NEON::BI__builtin_neon_vst1q_lane_v:
12287 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12288 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
12289 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
12290 return Builder.CreateAlignedStore(Ops[1], Builder.CreateBitCast(Ops[0], Ty),
12292 case NEON::BI__builtin_neon_vstl1_lane_s64:
12293 case NEON::BI__builtin_neon_vstl1q_lane_s64: {
12294 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12295 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
12296 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
12297 llvm::StoreInst *SI = Builder.CreateAlignedStore(
12298 Ops[1], Builder.CreateBitCast(Ops[0], Ty), PtrOp0.
getAlignment());
12299 SI->setAtomic(llvm::AtomicOrdering::Release);
12302 case NEON::BI__builtin_neon_vld2_v:
12303 case NEON::BI__builtin_neon_vld2q_v: {
12304 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
12305 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12306 llvm::Type *Tys[2] = { VTy, PTy };
12308 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
12309 Ops[0] = Builder.CreateBitCast(Ops[0],
12310 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12311 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12313 case NEON::BI__builtin_neon_vld3_v:
12314 case NEON::BI__builtin_neon_vld3q_v: {
12315 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
12316 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12317 llvm::Type *Tys[2] = { VTy, PTy };
12319 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
12320 Ops[0] = Builder.CreateBitCast(Ops[0],
12321 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12322 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12324 case NEON::BI__builtin_neon_vld4_v:
12325 case NEON::BI__builtin_neon_vld4q_v: {
12326 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
12327 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12328 llvm::Type *Tys[2] = { VTy, PTy };
12330 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
12331 Ops[0] = Builder.CreateBitCast(Ops[0],
12332 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12333 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12335 case NEON::BI__builtin_neon_vld2_dup_v:
12336 case NEON::BI__builtin_neon_vld2q_dup_v: {
12338 llvm::PointerType::getUnqual(VTy->getElementType());
12339 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12340 llvm::Type *Tys[2] = { VTy, PTy };
12342 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
12343 Ops[0] = Builder.CreateBitCast(Ops[0],
12344 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12345 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12347 case NEON::BI__builtin_neon_vld3_dup_v:
12348 case NEON::BI__builtin_neon_vld3q_dup_v: {
12350 llvm::PointerType::getUnqual(VTy->getElementType());
12351 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12352 llvm::Type *Tys[2] = { VTy, PTy };
12354 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
12355 Ops[0] = Builder.CreateBitCast(Ops[0],
12356 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12357 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12359 case NEON::BI__builtin_neon_vld4_dup_v:
12360 case NEON::BI__builtin_neon_vld4q_dup_v: {
12362 llvm::PointerType::getUnqual(VTy->getElementType());
12363 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
12364 llvm::Type *Tys[2] = { VTy, PTy };
12366 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
12367 Ops[0] = Builder.CreateBitCast(Ops[0],
12368 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12369 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12371 case NEON::BI__builtin_neon_vld2_lane_v:
12372 case NEON::BI__builtin_neon_vld2q_lane_v: {
12373 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
12374 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
12375 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
12376 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12377 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12378 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
12379 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld2_lane");
12380 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
12381 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12382 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12384 case NEON::BI__builtin_neon_vld3_lane_v:
12385 case NEON::BI__builtin_neon_vld3q_lane_v: {
12386 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
12387 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
12388 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
12389 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12390 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12391 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
12392 Ops[4] = Builder.CreateZExt(Ops[4],
Int64Ty);
12393 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld3_lane");
12394 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
12395 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12396 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12398 case NEON::BI__builtin_neon_vld4_lane_v:
12399 case NEON::BI__builtin_neon_vld4q_lane_v: {
12400 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
12401 Function *F =
CGM.
getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
12402 std::rotate(Ops.begin() + 1, Ops.begin() + 2, Ops.end());
12403 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12404 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12405 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
12406 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
12407 Ops[5] = Builder.CreateZExt(Ops[5],
Int64Ty);
12408 Ops[1] = Builder.CreateCall(F,
ArrayRef(Ops).slice(1),
"vld4_lane");
12409 Ty = llvm::PointerType::getUnqual(Ops[1]->
getType());
12410 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
12411 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
12413 case NEON::BI__builtin_neon_vst2_v:
12414 case NEON::BI__builtin_neon_vst2q_v: {
12415 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12416 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
12420 case NEON::BI__builtin_neon_vst2_lane_v:
12421 case NEON::BI__builtin_neon_vst2q_lane_v: {
12422 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12423 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
12424 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
12428 case NEON::BI__builtin_neon_vst3_v:
12429 case NEON::BI__builtin_neon_vst3q_v: {
12430 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12431 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
12435 case NEON::BI__builtin_neon_vst3_lane_v:
12436 case NEON::BI__builtin_neon_vst3q_lane_v: {
12437 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12438 Ops[3] = Builder.CreateZExt(Ops[3],
Int64Ty);
12439 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
12443 case NEON::BI__builtin_neon_vst4_v:
12444 case NEON::BI__builtin_neon_vst4q_v: {
12445 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12446 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
12450 case NEON::BI__builtin_neon_vst4_lane_v:
12451 case NEON::BI__builtin_neon_vst4q_lane_v: {
12452 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
12453 Ops[4] = Builder.CreateZExt(Ops[4],
Int64Ty);
12454 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
12458 case NEON::BI__builtin_neon_vtrn_v:
12459 case NEON::BI__builtin_neon_vtrnq_v: {
12460 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
12461 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12462 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12463 Value *SV =
nullptr;
12465 for (
unsigned vi = 0; vi != 2; ++vi) {
12467 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
12468 Indices.push_back(i+vi);
12469 Indices.push_back(i+e+vi);
12471 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
12472 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
12473 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
12477 case NEON::BI__builtin_neon_vuzp_v:
12478 case NEON::BI__builtin_neon_vuzpq_v: {
12479 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
12480 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12481 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12482 Value *SV =
nullptr;
12484 for (
unsigned vi = 0; vi != 2; ++vi) {
12486 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
12487 Indices.push_back(2*i+vi);
12489 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
12490 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
12491 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
12495 case NEON::BI__builtin_neon_vzip_v:
12496 case NEON::BI__builtin_neon_vzipq_v: {
12497 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
12498 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
12499 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
12500 Value *SV =
nullptr;
12502 for (
unsigned vi = 0; vi != 2; ++vi) {
12504 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
12505 Indices.push_back((i + vi*e) >> 1);
12506 Indices.push_back(((i + vi*e) >> 1)+e);
12508 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
12509 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
12510 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
12514 case NEON::BI__builtin_neon_vqtbl1q_v: {
12518 case NEON::BI__builtin_neon_vqtbl2q_v: {
12522 case NEON::BI__builtin_neon_vqtbl3q_v: {
12526 case NEON::BI__builtin_neon_vqtbl4q_v: {
12530 case NEON::BI__builtin_neon_vqtbx1q_v: {
12534 case NEON::BI__builtin_neon_vqtbx2q_v: {
12538 case NEON::BI__builtin_neon_vqtbx3q_v: {
12542 case NEON::BI__builtin_neon_vqtbx4q_v: {
12546 case NEON::BI__builtin_neon_vsqadd_v:
12547 case NEON::BI__builtin_neon_vsqaddq_v: {
12548 Int = Intrinsic::aarch64_neon_usqadd;
12551 case NEON::BI__builtin_neon_vuqadd_v:
12552 case NEON::BI__builtin_neon_vuqaddq_v: {
12553 Int = Intrinsic::aarch64_neon_suqadd;
12561 assert((BuiltinID == BPF::BI__builtin_preserve_field_info ||
12562 BuiltinID == BPF::BI__builtin_btf_type_id ||
12563 BuiltinID == BPF::BI__builtin_preserve_type_info ||
12564 BuiltinID == BPF::BI__builtin_preserve_enum_value) &&
12565 "unexpected BPF builtin");
12570 static uint32_t BuiltinSeqNum;
12572 switch (BuiltinID) {
12574 llvm_unreachable(
"Unexpected BPF builtin");
12575 case BPF::BI__builtin_preserve_field_info: {
12581 "using __builtin_preserve_field_info() without -g");
12594 Value *InfoKind = ConstantInt::get(
Int64Ty,
C->getSExtValue());
12597 llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration(
12598 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_field_info,
12599 {FieldAddr->getType()});
12600 return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind});
12602 case BPF::BI__builtin_btf_type_id:
12603 case BPF::BI__builtin_preserve_type_info: {
12614 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
12615 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
12617 llvm::Function *FnDecl;
12618 if (BuiltinID == BPF::BI__builtin_btf_type_id)
12619 FnDecl = llvm::Intrinsic::getDeclaration(
12620 &
CGM.
getModule(), llvm::Intrinsic::bpf_btf_type_id, {});
12622 FnDecl = llvm::Intrinsic::getDeclaration(
12623 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_type_info, {});
12624 CallInst *Fn = Builder.CreateCall(FnDecl, {SeqNumVal, FlagValue});
12625 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12628 case BPF::BI__builtin_preserve_enum_value: {
12644 auto &InitVal = Enumerator->getInitVal();
12645 std::string InitValStr;
12646 if (InitVal.isNegative() || InitVal > uint64_t(INT64_MAX))
12647 InitValStr = std::to_string(InitVal.getSExtValue());
12649 InitValStr = std::to_string(InitVal.getZExtValue());
12650 std::string EnumStr = Enumerator->getNameAsString() +
":" + InitValStr;
12651 Value *EnumStrVal = Builder.CreateGlobalStringPtr(EnumStr);
12654 Value *FlagValue = ConstantInt::get(
Int64Ty, Flag->getSExtValue());
12655 Value *SeqNumVal = ConstantInt::get(
Int32Ty, BuiltinSeqNum++);
12657 llvm::Function *IntrinsicFn = llvm::Intrinsic::getDeclaration(
12658 &
CGM.
getModule(), llvm::Intrinsic::bpf_preserve_enum_value, {});
12660 Builder.CreateCall(IntrinsicFn, {SeqNumVal, EnumStrVal, FlagValue});
12661 Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
12669 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
12670 "Not a power-of-two sized vector!");
12671 bool AllConstants =
true;
12672 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
12676 if (AllConstants) {
12678 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
12679 CstOps.push_back(cast<Constant>(Ops[i]));
12680 return llvm::ConstantVector::get(CstOps);
12685 llvm::FixedVectorType::get(Ops[0]->
getType(), Ops.size()));
12687 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
12688 Result = Builder.CreateInsertElement(
Result, Ops[i], Builder.getInt64(i));
12695 unsigned NumElts) {
12697 auto *MaskTy = llvm::FixedVectorType::get(
12699 cast<IntegerType>(Mask->
getType())->getBitWidth());
12700 Value *MaskVec = CGF.
Builder.CreateBitCast(Mask, MaskTy);
12706 for (
unsigned i = 0; i != NumElts; ++i)
12708 MaskVec = CGF.
Builder.CreateShuffleVector(
12709 MaskVec, MaskVec,
ArrayRef(Indices, NumElts),
"extract");
12718 llvm::PointerType::getUnqual(Ops[1]->
getType()));
12722 cast<llvm::FixedVectorType>(Ops[1]->
getType())->getNumElements());
12724 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Alignment, MaskVec);
12730 llvm::Type *Ty = Ops[1]->getType();
12732 CGF.
Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
12735 CGF, Ops[2], cast<llvm::FixedVectorType>(Ty)->getNumElements());
12737 return CGF.
Builder.CreateMaskedLoad(Ty, Ptr, Alignment, MaskVec, Ops[1]);
12743 llvm::Type *PtrTy = ResultTy->getElementType();
12747 llvm::PointerType::getUnqual(PtrTy));
12750 CGF, Ops[2], cast<FixedVectorType>(ResultTy)->getNumElements());
12752 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
12754 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
12764 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
12765 : Intrinsic::x86_avx512_mask_expand;
12767 return CGF.
Builder.CreateCall(F, { Ops[0], Ops[1], MaskVec });
12773 llvm::Type *PtrTy = ResultTy->getElementType();
12777 llvm::PointerType::getUnqual(PtrTy));
12781 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
12783 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
12788 bool InvertLHS =
false) {
12789 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
12794 LHS = CGF.
Builder.CreateNot(LHS);
12796 return CGF.
Builder.CreateBitCast(CGF.
Builder.CreateBinOp(Opc, LHS, RHS),
12797 Ops[0]->getType());
12801 Value *Amt,
bool IsRight) {
12802 llvm::Type *Ty = Op0->
getType();
12809 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
12810 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
12813 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
12815 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
12820 Value *Op0 = Ops[0];
12821 Value *Op1 = Ops[1];
12822 llvm::Type *Ty = Op0->
getType();
12825 CmpInst::Predicate Pred;
12828 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
12831 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
12834 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
12837 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
12840 Pred = ICmpInst::ICMP_EQ;
12843 Pred = ICmpInst::ICMP_NE;
12846 return llvm::Constant::getNullValue(Ty);
12848 return llvm::Constant::getAllOnesValue(Ty);
12850 llvm_unreachable(
"Unexpected XOP vpcom/vpcomu predicate");
12862 if (
const auto *
C = dyn_cast<Constant>(Mask))
12863 if (
C->isAllOnesValue())
12867 CGF, Mask, cast<llvm::FixedVectorType>(Op0->
getType())->getNumElements());
12869 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
12875 if (
const auto *
C = dyn_cast<Constant>(Mask))
12876 if (
C->isAllOnesValue())
12879 auto *MaskTy = llvm::FixedVectorType::get(
12880 CGF.
Builder.getInt1Ty(), Mask->
getType()->getIntegerBitWidth());
12881 Mask = CGF.
Builder.CreateBitCast(Mask, MaskTy);
12882 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
12883 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
12887 unsigned NumElts,
Value *MaskIn) {
12889 const auto *
C = dyn_cast<Constant>(MaskIn);
12890 if (!
C || !
C->isAllOnesValue())
12896 for (
unsigned i = 0; i != NumElts; ++i)
12898 for (
unsigned i = NumElts; i != 8; ++i)
12899 Indices[i] = i % NumElts + NumElts;
12900 Cmp = CGF.
Builder.CreateShuffleVector(
12901 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
12904 return CGF.
Builder.CreateBitCast(Cmp,
12906 std::max(NumElts, 8U)));
12911 assert((Ops.size() == 2 || Ops.size() == 4) &&
12912 "Unexpected number of arguments");
12918 Cmp = Constant::getNullValue(
12919 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
12920 }
else if (CC == 7) {
12921 Cmp = Constant::getAllOnesValue(
12922 llvm::FixedVectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
12924 ICmpInst::Predicate Pred;
12926 default: llvm_unreachable(
"Unknown condition code");
12927 case 0: Pred = ICmpInst::ICMP_EQ;
break;
12928 case 1: Pred =
Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
12929 case 2: Pred =
Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
12930 case 4: Pred = ICmpInst::ICMP_NE;
break;
12931 case 5: Pred =
Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
12932 case 6: Pred =
Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
12934 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
12937 Value *MaskIn =
nullptr;
12938 if (Ops.size() == 4)
12952 llvm::Type *Ty = Ops[1]->getType();
12956 Intrinsic::ID IID = IsSigned ? Intrinsic::x86_avx512_sitofp_round
12957 : Intrinsic::x86_avx512_uitofp_round;
12959 Res = CGF.
Builder.CreateCall(F, { Ops[0], Ops[3] });
12961 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
12962 Res = IsSigned ? CGF.
Builder.CreateSIToFP(Ops[0], Ty)
12963 : CGF.
Builder.CreateUIToFP(Ops[0], Ty);
12974 bool Subtract =
false;
12975 Intrinsic::ID IID = Intrinsic::not_intrinsic;
12976 switch (BuiltinID) {
12978 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
12981 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
12982 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
12983 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
12984 IID = llvm::Intrinsic::x86_avx512fp16_vfmadd_ph_512;
12986 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
12989 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
12990 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
12991 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
12992 IID = llvm::Intrinsic::x86_avx512fp16_vfmaddsub_ph_512;
12994 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
12997 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
12998 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
12999 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13000 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
13001 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13004 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13005 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13006 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13007 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
13008 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13011 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13012 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13013 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13014 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
13016 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13019 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13020 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13021 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13022 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
13036 if (IID != Intrinsic::not_intrinsic &&
13037 (cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4 ||
13040 Res = CGF.
Builder.CreateCall(Intr, {A, B,
C, Ops.back() });
13042 llvm::Type *Ty = A->
getType();
13044 if (CGF.
Builder.getIsFPConstrained()) {
13045 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13046 FMA = CGF.
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, Ty);
13047 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, {A, B,
C});
13050 Res = CGF.
Builder.CreateCall(FMA, {A, B,
C});
13055 Value *MaskFalseVal =
nullptr;
13056 switch (BuiltinID) {
13057 case clang::X86::BI__builtin_ia32_vfmaddph512_mask:
13058 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
13059 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
13060 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask:
13061 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
13062 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13063 MaskFalseVal = Ops[0];
13065 case clang::X86::BI__builtin_ia32_vfmaddph512_maskz:
13066 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
13067 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
13068 case clang::X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13069 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13070 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13071 MaskFalseVal = Constant::getNullValue(Ops[0]->
getType());
13073 case clang::X86::BI__builtin_ia32_vfmsubph512_mask3:
13074 case clang::X86::BI__builtin_ia32_vfmaddph512_mask3:
13075 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
13076 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
13077 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
13078 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
13079 case clang::X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13080 case clang::X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13081 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13082 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13083 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13084 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13085 MaskFalseVal = Ops[2];
13097 bool ZeroMask =
false,
unsigned PTIdx = 0,
13098 bool NegAcc =
false) {
13100 if (Ops.size() > 4)
13104 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
13106 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
13107 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
13108 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
13113 switch (Ops[0]->
getType()->getPrimitiveSizeInBits()) {
13115 IID = Intrinsic::x86_avx512fp16_vfmadd_f16;
13118 IID = Intrinsic::x86_avx512_vfmadd_f32;
13121 IID = Intrinsic::x86_avx512_vfmadd_f64;
13124 llvm_unreachable(
"Unexpected size");
13127 {Ops[0], Ops[1], Ops[2], Ops[4]});
13128 }
else if (CGF.
Builder.getIsFPConstrained()) {
13129 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
13131 Intrinsic::experimental_constrained_fma, Ops[0]->
getType());
13132 Res = CGF.
Builder.CreateConstrainedFPCall(FMA, Ops.slice(0, 3));
13135 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
13138 if (Ops.size() > 3) {
13139 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->
getType())
13145 if (NegAcc && PTIdx == 2)
13146 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
13150 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
13155 llvm::Type *Ty = Ops[0]->getType();
13157 Ty = llvm::FixedVectorType::get(CGF.
Int64Ty,
13158 Ty->getPrimitiveSizeInBits() / 64);
13164 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
13165 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
13166 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
13167 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
13168 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
13171 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
13172 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
13173 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
13176 return CGF.
Builder.CreateMul(LHS, RHS);
13184 llvm::Type *Ty = Ops[0]->getType();
13186 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
13187 unsigned EltWidth = Ty->getScalarSizeInBits();
13189 if (VecWidth == 128 && EltWidth == 32)
13190 IID = Intrinsic::x86_avx512_pternlog_d_128;
13191 else if (VecWidth == 256 && EltWidth == 32)
13192 IID = Intrinsic::x86_avx512_pternlog_d_256;
13193 else if (VecWidth == 512 && EltWidth == 32)
13194 IID = Intrinsic::x86_avx512_pternlog_d_512;
13195 else if (VecWidth == 128 && EltWidth == 64)
13196 IID = Intrinsic::x86_avx512_pternlog_q_128;
13197 else if (VecWidth == 256 && EltWidth == 64)
13198 IID = Intrinsic::x86_avx512_pternlog_q_256;
13199 else if (VecWidth == 512 && EltWidth == 64)
13200 IID = Intrinsic::x86_avx512_pternlog_q_512;
13202 llvm_unreachable(
"Unexpected intrinsic");
13206 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
13211 llvm::Type *DstTy) {
13212 unsigned NumberOfElements =
13215 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
13221 return EmitX86CpuIs(CPUStr);
13227 llvm::Type *DstTy) {
13228 assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
13229 "Unknown cvtph2ps intrinsic");
13232 if (Ops.size() == 4 && cast<llvm::ConstantInt>(Ops[3])->getZExtValue() != 4) {
13235 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1], Ops[2], Ops[3]});
13239 Value *Src = Ops[0];
13243 cast<llvm::FixedVectorType>(Src->
getType())->getNumElements()) {
13244 assert(NumDstElts == 4 &&
"Unexpected vector size");
13249 auto *HalfTy = llvm::FixedVectorType::get(
13251 Src = CGF.
Builder.CreateBitCast(Src, HalfTy);
13254 Value *Res = CGF.
Builder.CreateFPExt(Src, DstTy,
"cvtph2ps");
13256 if (Ops.size() >= 3)
13261Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
13263 llvm::Type *
Int32Ty = Builder.getInt32Ty();
13272 llvm::ArrayType::get(
Int32Ty, 1));
13282 std::tie(Index,
Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
13284 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
13286 .Case(
ALIAS, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
13288 .Case(STR, {1u,
static_cast<unsigned>(llvm::X86::ENUM)})
13290 .Case(
ALIAS, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
13292 .Case(STR, {2u,
static_cast<unsigned>(llvm::X86::ENUM)})
13293#include
"llvm/TargetParser/X86TargetParser.def"
13295 assert(
Value != 0 &&
"Invalid CPUStr passed to CpuIs");
13298 llvm::Value *Idxs[] = {ConstantInt::get(
Int32Ty, 0),
13299 ConstantInt::get(
Int32Ty, Index)};
13300 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
13301 CpuValue = Builder.CreateAlignedLoad(
Int32Ty, CpuValue,
13305 return Builder.CreateICmpEQ(CpuValue,
13309Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
13312 return EmitX86CpuSupports(FeatureStr);
13316 return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
13319llvm::Value *CodeGenFunction::EmitX86CpuSupports(uint64_t FeaturesMask) {
13320 uint32_t Features1 = Lo_32(FeaturesMask);
13321 uint32_t Features2 = Hi_32(FeaturesMask);
13325 if (Features1 != 0) {
13333 llvm::ArrayType::get(
Int32Ty, 1));
13341 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
13342 Builder.getInt32(0)};
13343 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
13344 Value *Features = Builder.CreateAlignedLoad(
Int32Ty, CpuFeatures,
13348 Value *Mask = Builder.getInt32(Features1);
13349 Value *Bitset = Builder.CreateAnd(Features, Mask);
13350 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
13354 if (Features2 != 0) {
13356 "__cpu_features2");
13359 Value *Features = Builder.CreateAlignedLoad(
Int32Ty, CpuFeatures2,
13363 Value *Mask = Builder.getInt32(Features2);
13364 Value *Bitset = Builder.CreateAnd(Features, Mask);
13365 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
13372Value *CodeGenFunction::EmitAArch64CpuInit() {
13373 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
13374 llvm::FunctionCallee Func =
13378 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
13379 return Builder.CreateCall(Func);
13382Value *CodeGenFunction::EmitX86CpuInit() {
13383 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
13385 llvm::FunctionCallee Func =
13389 ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
13390 return Builder.CreateCall(Func);
13395 uint64_t FeaturesMask = llvm::AArch64::getCpuSupportsMask(FeaturesStrs);
13397 if (FeaturesMask != 0) {
13402 llvm::Type *STy = llvm::StructType::get(
Int64Ty);
13403 llvm::Constant *AArch64CPUFeatures =
13406 llvm::Value *CpuFeatures = Builder.CreateGEP(
13407 STy, AArch64CPUFeatures,
13409 Value *Features = Builder.CreateAlignedLoad(
Int64Ty, CpuFeatures,
13411 Value *Mask = Builder.getInt64(FeaturesMask);
13412 Value *Bitset = Builder.CreateAnd(Features, Mask);
13413 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
13421 if (BuiltinID == X86::BI__builtin_cpu_is)
13422 return EmitX86CpuIs(E);
13423 if (BuiltinID == X86::BI__builtin_cpu_supports)
13424 return EmitX86CpuSupports(E);
13425 if (BuiltinID == X86::BI__builtin_cpu_init)
13426 return EmitX86CpuInit();
13434 bool IsMaskFCmp =
false;
13435 bool IsConjFMA =
false;
13438 unsigned ICEArguments = 0;
13443 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
13445 if ((ICEArguments & (1 << i)) == 0) {
13452 Ops.push_back(llvm::ConstantInt::get(
13462 auto getCmpIntrinsicCall = [
this, &Ops](Intrinsic::ID ID,
unsigned Imm) {
13463 Ops.push_back(llvm::ConstantInt::get(
Int8Ty, Imm));
13465 return Builder.CreateCall(F, Ops);
13473 auto getVectorFCmpIR = [
this, &Ops, E](CmpInst::Predicate Pred,
13474 bool IsSignaling) {
13475 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
13478 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
13480 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
13482 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
13483 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
13484 return Builder.CreateBitCast(Sext, FPVecTy);
13487 switch (BuiltinID) {
13488 default:
return nullptr;
13489 case X86::BI_mm_prefetch: {
13492 Value *RW = ConstantInt::get(
Int32Ty, (
C->getZExtValue() >> 2) & 0x1);
13493 Value *Locality = ConstantInt::get(
Int32Ty,
C->getZExtValue() & 0x3);
13496 return Builder.CreateCall(F, {
Address, RW, Locality,
Data});
13498 case X86::BI_mm_clflush: {
13499 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_clflush),
13502 case X86::BI_mm_lfence: {
13503 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_lfence));
13505 case X86::BI_mm_mfence: {
13506 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_mfence));
13508 case X86::BI_mm_sfence: {
13509 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse_sfence));
13511 case X86::BI_mm_pause: {
13512 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse2_pause));
13514 case X86::BI__rdtsc: {
13517 case X86::BI__builtin_ia32_rdtscp: {
13519 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
13521 return Builder.CreateExtractValue(Call, 0);
13523 case X86::BI__builtin_ia32_lzcnt_u16:
13524 case X86::BI__builtin_ia32_lzcnt_u32:
13525 case X86::BI__builtin_ia32_lzcnt_u64: {
13527 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
13529 case X86::BI__builtin_ia32_tzcnt_u16:
13530 case X86::BI__builtin_ia32_tzcnt_u32:
13531 case X86::BI__builtin_ia32_tzcnt_u64: {
13533 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
13535 case X86::BI__builtin_ia32_undef128:
13536 case X86::BI__builtin_ia32_undef256:
13537 case X86::BI__builtin_ia32_undef512:
13544 case X86::BI__builtin_ia32_vec_init_v8qi:
13545 case X86::BI__builtin_ia32_vec_init_v4hi:
13546 case X86::BI__builtin_ia32_vec_init_v2si:
13549 case X86::BI__builtin_ia32_vec_ext_v2si:
13550 case X86::BI__builtin_ia32_vec_ext_v16qi:
13551 case X86::BI__builtin_ia32_vec_ext_v8hi:
13552 case X86::BI__builtin_ia32_vec_ext_v4si:
13553 case X86::BI__builtin_ia32_vec_ext_v4sf:
13554 case X86::BI__builtin_ia32_vec_ext_v2di:
13555 case X86::BI__builtin_ia32_vec_ext_v32qi:
13556 case X86::BI__builtin_ia32_vec_ext_v16hi:
13557 case X86::BI__builtin_ia32_vec_ext_v8si:
13558 case X86::BI__builtin_ia32_vec_ext_v4di: {
13562 Index &= NumElts - 1;
13565 return Builder.CreateExtractElement(Ops[0], Index);
13567 case X86::BI__builtin_ia32_vec_set_v16qi:
13568 case X86::BI__builtin_ia32_vec_set_v8hi:
13569 case X86::BI__builtin_ia32_vec_set_v4si:
13570 case X86::BI__builtin_ia32_vec_set_v2di:
13571 case X86::BI__builtin_ia32_vec_set_v32qi:
13572 case X86::BI__builtin_ia32_vec_set_v16hi:
13573 case X86::BI__builtin_ia32_vec_set_v8si:
13574 case X86::BI__builtin_ia32_vec_set_v4di: {
13578 Index &= NumElts - 1;
13581 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
13583 case X86::BI_mm_setcsr:
13584 case X86::BI__builtin_ia32_ldmxcsr: {
13586 Builder.CreateStore(Ops[0], Tmp);
13587 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
13590 case X86::BI_mm_getcsr:
13591 case X86::BI__builtin_ia32_stmxcsr: {
13595 return Builder.CreateLoad(Tmp,
"stmxcsr");
13597 case X86::BI__builtin_ia32_xsave:
13598 case X86::BI__builtin_ia32_xsave64:
13599 case X86::BI__builtin_ia32_xrstor:
13600 case X86::BI__builtin_ia32_xrstor64:
13601 case X86::BI__builtin_ia32_xsaveopt:
13602 case X86::BI__builtin_ia32_xsaveopt64:
13603 case X86::BI__builtin_ia32_xrstors:
13604 case X86::BI__builtin_ia32_xrstors64:
13605 case X86::BI__builtin_ia32_xsavec:
13606 case X86::BI__builtin_ia32_xsavec64:
13607 case X86::BI__builtin_ia32_xsaves:
13608 case X86::BI__builtin_ia32_xsaves64:
13609 case X86::BI__builtin_ia32_xsetbv:
13610 case X86::BI_xsetbv: {
13612#define INTRINSIC_X86_XSAVE_ID(NAME) \
13613 case X86::BI__builtin_ia32_##NAME: \
13614 ID = Intrinsic::x86_##NAME; \
13616 switch (BuiltinID) {
13617 default: llvm_unreachable(
"Unsupported intrinsic!");
13631 case X86::BI_xsetbv:
13632 ID = Intrinsic::x86_xsetbv;
13635#undef INTRINSIC_X86_XSAVE_ID
13636 Value *Mhi = Builder.CreateTrunc(
13637 Builder.CreateLShr(Ops[1], ConstantInt::get(
Int64Ty, 32)),
Int32Ty);
13640 Ops.push_back(Mlo);
13643 case X86::BI__builtin_ia32_xgetbv:
13644 case X86::BI_xgetbv:
13645 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::x86_xgetbv), Ops);
13646 case X86::BI__builtin_ia32_storedqudi128_mask:
13647 case X86::BI__builtin_ia32_storedqusi128_mask:
13648 case X86::BI__builtin_ia32_storedquhi128_mask:
13649 case X86::BI__builtin_ia32_storedquqi128_mask:
13650 case X86::BI__builtin_ia32_storeupd128_mask:
13651 case X86::BI__builtin_ia32_storeups128_mask:
13652 case X86::BI__builtin_ia32_storedqudi256_mask:
13653 case X86::BI__builtin_ia32_storedqusi256_mask:
13654 case X86::BI__builtin_ia32_storedquhi256_mask:
13655 case X86::BI__builtin_ia32_storedquqi256_mask:
13656 case X86::BI__builtin_ia32_storeupd256_mask:
13657 case X86::BI__builtin_ia32_storeups256_mask:
13658 case X86::BI__builtin_ia32_storedqudi512_mask:
13659 case X86::BI__builtin_ia32_storedqusi512_mask:
13660 case X86::BI__builtin_ia32_storedquhi512_mask:
13661 case X86::BI__builtin_ia32_storedquqi512_mask:
13662 case X86::BI__builtin_ia32_storeupd512_mask:
13663 case X86::BI__builtin_ia32_storeups512_mask:
13666 case X86::BI__builtin_ia32_storesh128_mask:
13667 case X86::BI__builtin_ia32_storess128_mask:
13668 case X86::BI__builtin_ia32_storesd128_mask:
13671 case X86::BI__builtin_ia32_vpopcntb_128:
13672 case X86::BI__builtin_ia32_vpopcntd_128:
13673 case X86::BI__builtin_ia32_vpopcntq_128:
13674 case X86::BI__builtin_ia32_vpopcntw_128:
13675 case X86::BI__builtin_ia32_vpopcntb_256:
13676 case X86::BI__builtin_ia32_vpopcntd_256:
13677 case X86::BI__builtin_ia32_vpopcntq_256:
13678 case X86::BI__builtin_ia32_vpopcntw_256:
13679 case X86::BI__builtin_ia32_vpopcntb_512:
13680 case X86::BI__builtin_ia32_vpopcntd_512:
13681 case X86::BI__builtin_ia32_vpopcntq_512:
13682 case X86::BI__builtin_ia32_vpopcntw_512: {
13685 return Builder.CreateCall(F, Ops);
13687 case X86::BI__builtin_ia32_cvtmask2b128:
13688 case X86::BI__builtin_ia32_cvtmask2b256:
13689 case X86::BI__builtin_ia32_cvtmask2b512:
13690 case X86::BI__builtin_ia32_cvtmask2w128:
13691 case X86::BI__builtin_ia32_cvtmask2w256:
13692 case X86::BI__builtin_ia32_cvtmask2w512:
13693 case X86::BI__builtin_ia32_cvtmask2d128:
13694 case X86::BI__builtin_ia32_cvtmask2d256:
13695 case X86::BI__builtin_ia32_cvtmask2d512:
13696 case X86::BI__builtin_ia32_cvtmask2q128:
13697 case X86::BI__builtin_ia32_cvtmask2q256:
13698 case X86::BI__builtin_ia32_cvtmask2q512:
13701 case X86::BI__builtin_ia32_cvtb2mask128:
13702 case X86::BI__builtin_ia32_cvtb2mask256:
13703 case X86::BI__builtin_ia32_cvtb2mask512:
13704 case X86::BI__builtin_ia32_cvtw2mask128:
13705 case X86::BI__builtin_ia32_cvtw2mask256:
13706 case X86::BI__builtin_ia32_cvtw2mask512:
13707 case X86::BI__builtin_ia32_cvtd2mask128:
13708 case X86::BI__builtin_ia32_cvtd2mask256:
13709 case X86::BI__builtin_ia32_cvtd2mask512:
13710 case X86::BI__builtin_ia32_cvtq2mask128:
13711 case X86::BI__builtin_ia32_cvtq2mask256:
13712 case X86::BI__builtin_ia32_cvtq2mask512:
13715 case X86::BI__builtin_ia32_cvtdq2ps512_mask:
13716 case X86::BI__builtin_ia32_cvtqq2ps512_mask:
13717 case X86::BI__builtin_ia32_cvtqq2pd512_mask:
13718 case X86::BI__builtin_ia32_vcvtw2ph512_mask:
13719 case X86::BI__builtin_ia32_vcvtdq2ph512_mask:
13720 case X86::BI__builtin_ia32_vcvtqq2ph512_mask:
13722 case X86::BI__builtin_ia32_cvtudq2ps512_mask:
13723 case X86::BI__builtin_ia32_cvtuqq2ps512_mask:
13724 case X86::BI__builtin_ia32_cvtuqq2pd512_mask:
13725 case X86::BI__builtin_ia32_vcvtuw2ph512_mask:
13726 case X86::BI__builtin_ia32_vcvtudq2ph512_mask:
13727 case X86::BI__builtin_ia32_vcvtuqq2ph512_mask:
13730 case X86::BI__builtin_ia32_vfmaddss3:
13731 case X86::BI__builtin_ia32_vfmaddsd3:
13732 case X86::BI__builtin_ia32_vfmaddsh3_mask:
13733 case X86::BI__builtin_ia32_vfmaddss3_mask:
13734 case X86::BI__builtin_ia32_vfmaddsd3_mask:
13736 case X86::BI__builtin_ia32_vfmaddss:
13737 case X86::BI__builtin_ia32_vfmaddsd:
13739 Constant::getNullValue(Ops[0]->
getType()));
13740 case X86::BI__builtin_ia32_vfmaddsh3_maskz:
13741 case X86::BI__builtin_ia32_vfmaddss3_maskz:
13742 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
13744 case X86::BI__builtin_ia32_vfmaddsh3_mask3:
13745 case X86::BI__builtin_ia32_vfmaddss3_mask3:
13746 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
13748 case X86::BI__builtin_ia32_vfmsubsh3_mask3:
13749 case X86::BI__builtin_ia32_vfmsubss3_mask3:
13750 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
13753 case X86::BI__builtin_ia32_vfmaddph:
13754 case X86::BI__builtin_ia32_vfmaddps:
13755 case X86::BI__builtin_ia32_vfmaddpd:
13756 case X86::BI__builtin_ia32_vfmaddph256:
13757 case X86::BI__builtin_ia32_vfmaddps256:
13758 case X86::BI__builtin_ia32_vfmaddpd256:
13759 case X86::BI__builtin_ia32_vfmaddph512_mask:
13760 case X86::BI__builtin_ia32_vfmaddph512_maskz:
13761 case X86::BI__builtin_ia32_vfmaddph512_mask3:
13762 case X86::BI__builtin_ia32_vfmaddps512_mask:
13763 case X86::BI__builtin_ia32_vfmaddps512_maskz:
13764 case X86::BI__builtin_ia32_vfmaddps512_mask3:
13765 case X86::BI__builtin_ia32_vfmsubps512_mask3:
13766 case X86::BI__builtin_ia32_vfmaddpd512_mask:
13767 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
13768 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
13769 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
13770 case X86::BI__builtin_ia32_vfmsubph512_mask3:
13772 case X86::BI__builtin_ia32_vfmaddsubph512_mask:
13773 case X86::BI__builtin_ia32_vfmaddsubph512_maskz:
13774 case X86::BI__builtin_ia32_vfmaddsubph512_mask3:
13775 case X86::BI__builtin_ia32_vfmsubaddph512_mask3:
13776 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
13777 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
13778 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
13779 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
13780 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
13781 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
13782 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
13783 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
13786 case X86::BI__builtin_ia32_movdqa32store128_mask:
13787 case X86::BI__builtin_ia32_movdqa64store128_mask:
13788 case X86::BI__builtin_ia32_storeaps128_mask:
13789 case X86::BI__builtin_ia32_storeapd128_mask:
13790 case X86::BI__builtin_ia32_movdqa32store256_mask:
13791 case X86::BI__builtin_ia32_movdqa64store256_mask:
13792 case X86::BI__builtin_ia32_storeaps256_mask:
13793 case X86::BI__builtin_ia32_storeapd256_mask:
13794 case X86::BI__builtin_ia32_movdqa32store512_mask:
13795 case X86::BI__builtin_ia32_movdqa64store512_mask:
13796 case X86::BI__builtin_ia32_storeaps512_mask:
13797 case X86::BI__builtin_ia32_storeapd512_mask:
13802 case X86::BI__builtin_ia32_loadups128_mask:
13803 case X86::BI__builtin_ia32_loadups256_mask:
13804 case X86::BI__builtin_ia32_loadups512_mask:
13805 case X86::BI__builtin_ia32_loadupd128_mask:
13806 case X86::BI__builtin_ia32_loadupd256_mask:
13807 case X86::BI__builtin_ia32_loadupd512_mask:
13808 case X86::BI__builtin_ia32_loaddquqi128_mask:
13809 case X86::BI__builtin_ia32_loaddquqi256_mask:
13810 case X86::BI__builtin_ia32_loaddquqi512_mask:
13811 case X86::BI__builtin_ia32_loaddquhi128_mask:
13812 case X86::BI__builtin_ia32_loaddquhi256_mask:
13813 case X86::BI__builtin_ia32_loaddquhi512_mask:
13814 case X86::BI__builtin_ia32_loaddqusi128_mask:
13815 case X86::BI__builtin_ia32_loaddqusi256_mask:
13816 case X86::BI__builtin_ia32_loaddqusi512_mask:
13817 case X86::BI__builtin_ia32_loaddqudi128_mask:
13818 case X86::BI__builtin_ia32_loaddqudi256_mask:
13819 case X86::BI__builtin_ia32_loaddqudi512_mask:
13822 case X86::BI__builtin_ia32_loadsh128_mask:
13823 case X86::BI__builtin_ia32_loadss128_mask:
13824 case X86::BI__builtin_ia32_loadsd128_mask:
13827 case X86::BI__builtin_ia32_loadaps128_mask:
13828 case X86::BI__builtin_ia32_loadaps256_mask:
13829 case X86::BI__builtin_ia32_loadaps512_mask:
13830 case X86::BI__builtin_ia32_loadapd128_mask:
13831 case X86::BI__builtin_ia32_loadapd256_mask:
13832 case X86::BI__builtin_ia32_loadapd512_mask:
13833 case X86::BI__builtin_ia32_movdqa32load128_mask:
13834 case X86::BI__builtin_ia32_movdqa32load256_mask:
13835 case X86::BI__builtin_ia32_movdqa32load512_mask:
13836 case X86::BI__builtin_ia32_movdqa64load128_mask:
13837 case X86::BI__builtin_ia32_movdqa64load256_mask:
13838 case X86::BI__builtin_ia32_movdqa64load512_mask:
13843 case X86::BI__builtin_ia32_expandloaddf128_mask:
13844 case X86::BI__builtin_ia32_expandloaddf256_mask:
13845 case X86::BI__builtin_ia32_expandloaddf512_mask:
13846 case X86::BI__builtin_ia32_expandloadsf128_mask:
13847 case X86::BI__builtin_ia32_expandloadsf256_mask:
13848 case X86::BI__builtin_ia32_expandloadsf512_mask:
13849 case X86::BI__builtin_ia32_expandloaddi128_mask:
13850 case X86::BI__builtin_ia32_expandloaddi256_mask:
13851 case X86::BI__builtin_ia32_expandloaddi512_mask:
13852 case X86::BI__builtin_ia32_expandloadsi128_mask:
13853 case X86::BI__builtin_ia32_expandloadsi256_mask:
13854 case X86::BI__builtin_ia32_expandloadsi512_mask:
13855 case X86::BI__builtin_ia32_expandloadhi128_mask:
13856 case X86::BI__builtin_ia32_expandloadhi256_mask:
13857 case X86::BI__builtin_ia32_expandloadhi512_mask:
13858 case X86::BI__builtin_ia32_expandloadqi128_mask:
13859 case X86::BI__builtin_ia32_expandloadqi256_mask:
13860 case X86::BI__builtin_ia32_expandloadqi512_mask:
13863 case X86::BI__builtin_ia32_compressstoredf128_mask:
13864 case X86::BI__builtin_ia32_compressstoredf256_mask:
13865 case X86::BI__builtin_ia32_compressstoredf512_mask:
13866 case X86::BI__builtin_ia32_compressstoresf128_mask:
13867 case X86::BI__builtin_ia32_compressstoresf256_mask:
13868 case X86::BI__builtin_ia32_compressstoresf512_mask:
13869 case X86::BI__builtin_ia32_compressstoredi128_mask:
13870 case X86::BI__builtin_ia32_compressstoredi256_mask:
13871 case X86::BI__builtin_ia32_compressstoredi512_mask:
13872 case X86::BI__builtin_ia32_compressstoresi128_mask:
13873 case X86::BI__builtin_ia32_compressstoresi256_mask:
13874 case X86::BI__builtin_ia32_compressstoresi512_mask:
13875 case X86::BI__builtin_ia32_compressstorehi128_mask:
13876 case X86::BI__builtin_ia32_compressstorehi256_mask:
13877 case X86::BI__builtin_ia32_compressstorehi512_mask:
13878 case X86::BI__builtin_ia32_compressstoreqi128_mask:
13879 case X86::BI__builtin_ia32_compressstoreqi256_mask:
13880 case X86::BI__builtin_ia32_compressstoreqi512_mask:
13883 case X86::BI__builtin_ia32_expanddf128_mask:
13884 case X86::BI__builtin_ia32_expanddf256_mask:
13885 case X86::BI__builtin_ia32_expanddf512_mask:
13886 case X86::BI__builtin_ia32_expandsf128_mask:
13887 case X86::BI__builtin_ia32_expandsf256_mask:
13888 case X86::BI__builtin_ia32_expandsf512_mask:
13889 case X86::BI__builtin_ia32_expanddi128_mask:
13890 case X86::BI__builtin_ia32_expanddi256_mask:
13891 case X86::BI__builtin_ia32_expanddi512_mask:
13892 case X86::BI__builtin_ia32_expandsi128_mask:
13893 case X86::BI__builtin_ia32_expandsi256_mask:
13894 case X86::BI__builtin_ia32_expandsi512_mask:
13895 case X86::BI__builtin_ia32_expandhi128_mask:
13896 case X86::BI__builtin_ia32_expandhi256_mask:
13897 case X86::BI__builtin_ia32_expandhi512_mask:
13898 case X86::BI__builtin_ia32_expandqi128_mask:
13899 case X86::BI__builtin_ia32_expandqi256_mask:
13900 case X86::BI__builtin_ia32_expandqi512_mask:
13903 case X86::BI__builtin_ia32_compressdf128_mask:
13904 case X86::BI__builtin_ia32_compressdf256_mask:
13905 case X86::BI__builtin_ia32_compressdf512_mask:
13906 case X86::BI__builtin_ia32_compresssf128_mask:
13907 case X86::BI__builtin_ia32_compresssf256_mask:
13908 case X86::BI__builtin_ia32_compresssf512_mask:
13909 case X86::BI__builtin_ia32_compressdi128_mask:
13910 case X86::BI__builtin_ia32_compressdi256_mask:
13911 case X86::BI__builtin_ia32_compressdi512_mask:
13912 case X86::BI__builtin_ia32_compresssi128_mask:
13913 case X86::BI__builtin_ia32_compresssi256_mask:
13914 case X86::BI__builtin_ia32_compresssi512_mask:
13915 case X86::BI__builtin_ia32_compresshi128_mask:
13916 case X86::BI__builtin_ia32_compresshi256_mask:
13917 case X86::BI__builtin_ia32_compresshi512_mask:
13918 case X86::BI__builtin_ia32_compressqi128_mask:
13919 case X86::BI__builtin_ia32_compressqi256_mask:
13920 case X86::BI__builtin_ia32_compressqi512_mask:
13923 case X86::BI__builtin_ia32_gather3div2df:
13924 case X86::BI__builtin_ia32_gather3div2di:
13925 case X86::BI__builtin_ia32_gather3div4df:
13926 case X86::BI__builtin_ia32_gather3div4di:
13927 case X86::BI__builtin_ia32_gather3div4sf:
13928 case X86::BI__builtin_ia32_gather3div4si:
13929 case X86::BI__builtin_ia32_gather3div8sf:
13930 case X86::BI__builtin_ia32_gather3div8si:
13931 case X86::BI__builtin_ia32_gather3siv2df:
13932 case X86::BI__builtin_ia32_gather3siv2di:
13933 case X86::BI__builtin_ia32_gather3siv4df:
13934 case X86::BI__builtin_ia32_gather3siv4di:
13935 case X86::BI__builtin_ia32_gather3siv4sf:
13936 case X86::BI__builtin_ia32_gather3siv4si:
13937 case X86::BI__builtin_ia32_gather3siv8sf:
13938 case X86::BI__builtin_ia32_gather3siv8si:
13939 case X86::BI__builtin_ia32_gathersiv8df:
13940 case X86::BI__builtin_ia32_gathersiv16sf:
13941 case X86::BI__builtin_ia32_gatherdiv8df:
13942 case X86::BI__builtin_ia32_gatherdiv16sf:
13943 case X86::BI__builtin_ia32_gathersiv8di:
13944 case X86::BI__builtin_ia32_gathersiv16si:
13945 case X86::BI__builtin_ia32_gatherdiv8di:
13946 case X86::BI__builtin_ia32_gatherdiv16si: {
13948 switch (BuiltinID) {
13949 default: llvm_unreachable(
"Unexpected builtin");
13950 case X86::BI__builtin_ia32_gather3div2df:
13951 IID = Intrinsic::x86_avx512_mask_gather3div2_df;
13953 case X86::BI__builtin_ia32_gather3div2di:
13954 IID = Intrinsic::x86_avx512_mask_gather3div2_di;
13956 case X86::BI__builtin_ia32_gather3div4df:
13957 IID = Intrinsic::x86_avx512_mask_gather3div4_df;
13959 case X86::BI__builtin_ia32_gather3div4di:
13960 IID = Intrinsic::x86_avx512_mask_gather3div4_di;
13962 case X86::BI__builtin_ia32_gather3div4sf:
13963 IID = Intrinsic::x86_avx512_mask_gather3div4_sf;
13965 case X86::BI__builtin_ia32_gather3div4si:
13966 IID = Intrinsic::x86_avx512_mask_gather3div4_si;
13968 case X86::BI__builtin_ia32_gather3div8sf:
13969 IID = Intrinsic::x86_avx512_mask_gather3div8_sf;
13971 case X86::BI__builtin_ia32_gather3div8si:
13972 IID = Intrinsic::x86_avx512_mask_gather3div8_si;
13974 case X86::BI__builtin_ia32_gather3siv2df:
13975 IID = Intrinsic::x86_avx512_mask_gather3siv2_df;
13977 case X86::BI__builtin_ia32_gather3siv2di:
13978 IID = Intrinsic::x86_avx512_mask_gather3siv2_di;
13980 case X86::BI__builtin_ia32_gather3siv4df:
13981 IID = Intrinsic::x86_avx512_mask_gather3siv4_df;
13983 case X86::BI__builtin_ia32_gather3siv4di:
13984 IID = Intrinsic::x86_avx512_mask_gather3siv4_di;
13986 case X86::BI__builtin_ia32_gather3siv4sf:
13987 IID = Intrinsic::x86_avx512_mask_gather3siv4_sf;
13989 case X86::BI__builtin_ia32_gather3siv4si:
13990 IID = Intrinsic::x86_avx512_mask_gather3siv4_si;
13992 case X86::BI__builtin_ia32_gather3siv8sf:
13993 IID = Intrinsic::x86_avx512_mask_gather3siv8_sf;
13995 case X86::BI__builtin_ia32_gather3siv8si:
13996 IID = Intrinsic::x86_avx512_mask_gather3siv8_si;
13998 case X86::BI__builtin_ia32_gathersiv8df:
13999 IID = Intrinsic::x86_avx512_mask_gather_dpd_512;
14001 case X86::BI__builtin_ia32_gathersiv16sf:
14002 IID = Intrinsic::x86_avx512_mask_gather_dps_512;
14004 case X86::BI__builtin_ia32_gatherdiv8df:
14005 IID = Intrinsic::x86_avx512_mask_gather_qpd_512;
14007 case X86::BI__builtin_ia32_gatherdiv16sf:
14008 IID = Intrinsic::x86_avx512_mask_gather_qps_512;
14010 case X86::BI__builtin_ia32_gathersiv8di:
14011 IID = Intrinsic::x86_avx512_mask_gather_dpq_512;
14013 case X86::BI__builtin_ia32_gathersiv16si:
14014 IID = Intrinsic::x86_avx512_mask_gather_dpi_512;
14016 case X86::BI__builtin_ia32_gatherdiv8di:
14017 IID = Intrinsic::x86_avx512_mask_gather_qpq_512;
14019 case X86::BI__builtin_ia32_gatherdiv16si:
14020 IID = Intrinsic::x86_avx512_mask_gather_qpi_512;
14024 unsigned MinElts = std::min(
14025 cast<llvm::FixedVectorType>(Ops[0]->
getType())->getNumElements(),
14026 cast<llvm::FixedVectorType>(Ops[2]->
getType())->getNumElements());
14029 return Builder.CreateCall(Intr, Ops);
14032 case X86::BI__builtin_ia32_scattersiv8df:
14033 case X86::BI__builtin_ia32_scattersiv16sf:
14034 case X86::BI__builtin_ia32_scatterdiv8df:
14035 case X86::BI__builtin_ia32_scatterdiv16sf:
14036 case X86::BI__builtin_ia32_scattersiv8di:
14037 case X86::BI__builtin_ia32_scattersiv16si:
14038 case X86::BI__builtin_ia32_scatterdiv8di:
14039 case X86::BI__builtin_ia32_scatterdiv16si:
14040 case X86::BI__builtin_ia32_scatterdiv2df:
14041 case X86::BI__builtin_ia32_scatterdiv2di:
14042 case X86::BI__builtin_ia32_scatterdiv4df:
14043 case X86::BI__builtin_ia32_scatterdiv4di:
14044 case X86::BI__builtin_ia32_scatterdiv4sf:
14045 case X86::BI__builtin_ia32_scatterdiv4si:
14046 case X86::BI__builtin_ia32_scatterdiv8sf:
14047 case X86::BI__builtin_ia32_scatterdiv8si:
14048 case X86::BI__builtin_ia32_scattersiv2df:
14049 case X86::BI__builtin_ia32_scattersiv2di:
14050 case X86::BI__builtin_ia32_scattersiv4df:
14051 case X86::BI__builtin_ia32_scattersiv4di:
14052 case X86::BI__builtin_ia32_scattersiv4sf:
14053 case X86::BI__builtin_ia32_scattersiv4si:
14054 case X86::BI__builtin_ia32_scattersiv8sf:
14055 case X86::BI__builtin_ia32_scattersiv8si: {
14057 switch (BuiltinID) {
14058 default: llvm_unreachable(
"Unexpected builtin");
14059 case X86::BI__builtin_ia32_scattersiv8df:
14060 IID = Intrinsic::x86_avx512_mask_scatter_dpd_512;
14062 case X86::BI__builtin_ia32_scattersiv16sf:
14063 IID = Intrinsic::x86_avx512_mask_scatter_dps_512;
14065 case X86::BI__builtin_ia32_scatterdiv8df:
14066 IID = Intrinsic::x86_avx512_mask_scatter_qpd_512;
14068 case X86::BI__builtin_ia32_scatterdiv16sf:
14069 IID = Intrinsic::x86_avx512_mask_scatter_qps_512;
14071 case X86::BI__builtin_ia32_scattersiv8di:
14072 IID = Intrinsic::x86_avx512_mask_scatter_dpq_512;
14074 case X86::BI__builtin_ia32_scattersiv16si:
14075 IID = Intrinsic::x86_avx512_mask_scatter_dpi_512;
14077 case X86::BI__builtin_ia32_scatterdiv8di:
14078 IID = Intrinsic::x86_avx512_mask_scatter_qpq_512;
14080 case X86::BI__builtin_ia32_scatterdiv16si:
14081 IID = Intrinsic::x86_avx512_mask_scatter_qpi_512;
14083 case X86::BI__builtin_ia32_scatterdiv2df:
14084 IID = Intrinsic::x86_avx512_mask_scatterdiv2_df;
14086 case X86::BI__builtin_ia32_scatterdiv2di:
14087 IID = Intrinsic::x86_avx512_mask_scatterdiv2_di;
14089 case X86::BI__builtin_ia32_scatterdiv4df:
14090 IID = Intrinsic::x86_avx512_mask_scatterdiv4_df;
14092 case X86::BI__builtin_ia32_scatterdiv4di:
14093 IID = Intrinsic::x86_avx512_mask_scatterdiv4_di;
14095 case X86::BI__builtin_ia32_scatterdiv4sf:
14096 IID = Intrinsic::x86_avx512_mask_scatterdiv4_sf;
14098 case X86::BI__builtin_ia32_scatterdiv4si:
14099 IID = Intrinsic::x86_avx512_mask_scatterdiv4_si;
14101 case X86::BI__builtin_ia32_scatterdiv8sf:
14102 IID = Intrinsic::x86_avx512_mask_scatterdiv8_sf;
14104 case X86::BI__builtin_ia32_scatterdiv8si:
14105 IID = Intrinsic::x86_avx512_mask_scatterdiv8_si;
14107 case X86::BI__builtin_ia32_scattersiv2df:
14108 IID = Intrinsic::x86_avx512_mask_scattersiv2_df;
14110 case X86::BI__builtin_ia32_scattersiv2di:
14111 IID = Intrinsic::x86_avx512_mask_scattersiv2_di;
14113 case X86::BI__builtin_ia32_scattersiv4df:
14114 IID = Intrinsic::x86_avx512_mask_scattersiv4_df;
14116 case X86::BI__builtin_ia32_scattersiv4di:
14117 IID = Intrinsic::x86_avx512_mask_scattersiv4_di;
14119 case X86::BI__builtin_ia32_scattersiv4sf:
14120 IID = Intrinsic::x86_avx512_mask_scattersiv4_sf;
14122 case X86::BI__builtin_ia32_scattersiv4si:
14123 IID = Intrinsic::x86_avx512_mask_scattersiv4_si;
14125 case X86::BI__builtin_ia32_scattersiv8sf:
14126 IID = Intrinsic::x86_avx512_mask_scattersiv8_sf;
14128 case X86::BI__builtin_ia32_scattersiv8si:
14129 IID = Intrinsic::x86_avx512_mask_scattersiv8_si;
14133 unsigned MinElts = std::min(
14134 cast<llvm::FixedVectorType>(Ops[2]->
getType())->getNumElements(),
14135 cast<llvm::FixedVectorType>(Ops[3]->
getType())->getNumElements());
14138 return Builder.CreateCall(Intr, Ops);
14141 case X86::BI__builtin_ia32_vextractf128_pd256:
14142 case X86::BI__builtin_ia32_vextractf128_ps256:
14143 case X86::BI__builtin_ia32_vextractf128_si256:
14144 case X86::BI__builtin_ia32_extract128i256:
14145 case X86::BI__builtin_ia32_extractf64x4_mask:
14146 case X86::BI__builtin_ia32_extractf32x4_mask:
14147 case X86::BI__builtin_ia32_extracti64x4_mask:
14148 case X86::BI__builtin_ia32_extracti32x4_mask:
14149 case X86::BI__builtin_ia32_extractf32x8_mask:
14150 case X86::BI__builtin_ia32_extracti32x8_mask:
14151 case X86::BI__builtin_ia32_extractf32x4_256_mask:
14152 case X86::BI__builtin_ia32_extracti32x4_256_mask:
14153 case X86::BI__builtin_ia32_extractf64x2_256_mask:
14154 case X86::BI__builtin_ia32_extracti64x2_256_mask:
14155 case X86::BI__builtin_ia32_extractf64x2_512_mask:
14156 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
14158 unsigned NumElts = DstTy->getNumElements();
14159 unsigned SrcNumElts =
14161 unsigned SubVectors = SrcNumElts / NumElts;
14163 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14164 Index &= SubVectors - 1;
14168 for (
unsigned i = 0; i != NumElts; ++i)
14169 Indices[i] = i + Index;
14171 Value *Res = Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
14174 if (Ops.size() == 4)
14179 case X86::BI__builtin_ia32_vinsertf128_pd256:
14180 case X86::BI__builtin_ia32_vinsertf128_ps256:
14181 case X86::BI__builtin_ia32_vinsertf128_si256:
14182 case X86::BI__builtin_ia32_insert128i256:
14183 case X86::BI__builtin_ia32_insertf64x4:
14184 case X86::BI__builtin_ia32_insertf32x4:
14185 case X86::BI__builtin_ia32_inserti64x4:
14186 case X86::BI__builtin_ia32_inserti32x4:
14187 case X86::BI__builtin_ia32_insertf32x8:
14188 case X86::BI__builtin_ia32_inserti32x8:
14189 case X86::BI__builtin_ia32_insertf32x4_256:
14190 case X86::BI__builtin_ia32_inserti32x4_256:
14191 case X86::BI__builtin_ia32_insertf64x2_256:
14192 case X86::BI__builtin_ia32_inserti64x2_256:
14193 case X86::BI__builtin_ia32_insertf64x2_512:
14194 case X86::BI__builtin_ia32_inserti64x2_512: {
14195 unsigned DstNumElts =
14197 unsigned SrcNumElts =
14199 unsigned SubVectors = DstNumElts / SrcNumElts;
14201 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
14202 Index &= SubVectors - 1;
14203 Index *= SrcNumElts;
14206 for (
unsigned i = 0; i != DstNumElts; ++i)
14207 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
14209 Value *Op1 = Builder.CreateShuffleVector(
14210 Ops[1],
ArrayRef(Indices, DstNumElts),
"widen");
14212 for (
unsigned i = 0; i != DstNumElts; ++i) {
14213 if (i >= Index && i < (Index + SrcNumElts))
14214 Indices[i] = (i - Index) + DstNumElts;
14219 return Builder.CreateShuffleVector(Ops[0], Op1,
14220 ArrayRef(Indices, DstNumElts),
"insert");
14222 case X86::BI__builtin_ia32_pmovqd512_mask:
14223 case X86::BI__builtin_ia32_pmovwb512_mask: {
14224 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->
getType());
14227 case X86::BI__builtin_ia32_pmovdb512_mask:
14228 case X86::BI__builtin_ia32_pmovdw512_mask:
14229 case X86::BI__builtin_ia32_pmovqw512_mask: {
14230 if (
const auto *
C = dyn_cast<Constant>(Ops[2]))
14231 if (
C->isAllOnesValue())
14232 return Builder.CreateTrunc(Ops[0], Ops[1]->
getType());
14235 switch (BuiltinID) {
14236 default: llvm_unreachable(
"Unsupported intrinsic!");
14237 case X86::BI__builtin_ia32_pmovdb512_mask:
14238 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
14240 case X86::BI__builtin_ia32_pmovdw512_mask:
14241 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
14243 case X86::BI__builtin_ia32_pmovqw512_mask:
14244 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
14249 return Builder.CreateCall(Intr, Ops);
14251 case X86::BI__builtin_ia32_pblendw128:
14252 case X86::BI__builtin_ia32_blendpd:
14253 case X86::BI__builtin_ia32_blendps:
14254 case X86::BI__builtin_ia32_blendpd256:
14255 case X86::BI__builtin_ia32_blendps256:
14256 case X86::BI__builtin_ia32_pblendw256:
14257 case X86::BI__builtin_ia32_pblendd128:
14258 case X86::BI__builtin_ia32_pblendd256: {
14266 for (
unsigned i = 0; i != NumElts; ++i)
14267 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
14269 return Builder.CreateShuffleVector(Ops[0], Ops[1],
14270 ArrayRef(Indices, NumElts),
"blend");
14272 case X86::BI__builtin_ia32_pshuflw:
14273 case X86::BI__builtin_ia32_pshuflw256:
14274 case X86::BI__builtin_ia32_pshuflw512: {
14277 unsigned NumElts = Ty->getNumElements();
14280 Imm = (Imm & 0xff) * 0x01010101;
14283 for (
unsigned l = 0; l != NumElts; l += 8) {
14284 for (
unsigned i = 0; i != 4; ++i) {
14285 Indices[l + i] = l + (Imm & 3);
14288 for (
unsigned i = 4; i != 8; ++i)
14289 Indices[l + i] = l + i;
14292 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
14295 case X86::BI__builtin_ia32_pshufhw:
14296 case X86::BI__builtin_ia32_pshufhw256:
14297 case X86::BI__builtin_ia32_pshufhw512: {
14300 unsigned NumElts = Ty->getNumElements();
14303 Imm = (Imm & 0xff) * 0x01010101;
14306 for (
unsigned l = 0; l != NumElts; l += 8) {
14307 for (
unsigned i = 0; i != 4; ++i)
14308 Indices[l + i] = l + i;
14309 for (
unsigned i = 4; i != 8; ++i) {
14310 Indices[l + i] = l + 4 + (Imm & 3);
14315 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
14318 case X86::BI__builtin_ia32_pshufd:
14319 case X86::BI__builtin_ia32_pshufd256:
14320 case X86::BI__builtin_ia32_pshufd512:
14321 case X86::BI__builtin_ia32_vpermilpd:
14322 case X86::BI__builtin_ia32_vpermilps:
14323 case X86::BI__builtin_ia32_vpermilpd256:
14324 case X86::BI__builtin_ia32_vpermilps256:
14325 case X86::BI__builtin_ia32_vpermilpd512:
14326 case X86::BI__builtin_ia32_vpermilps512: {
14329 unsigned NumElts = Ty->getNumElements();
14330 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
14331 unsigned NumLaneElts = NumElts / NumLanes;
14334 Imm = (Imm & 0xff) * 0x01010101;
14337 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
14338 for (
unsigned i = 0; i != NumLaneElts; ++i) {
14339 Indices[i + l] = (Imm % NumLaneElts) + l;
14340 Imm /= NumLaneElts;
14344 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
14347 case X86::BI__builtin_ia32_shufpd:
14348 case X86::BI__builtin_ia32_shufpd256:
14349 case X86::BI__builtin_ia32_shufpd512:
14350 case X86::BI__builtin_ia32_shufps:
14351 case X86::BI__builtin_ia32_shufps256:
14352 case X86::BI__builtin_ia32_shufps512: {
14355 unsigned NumElts = Ty->getNumElements();
14356 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
14357 unsigned NumLaneElts = NumElts / NumLanes;
14360 Imm = (Imm & 0xff) * 0x01010101;
14363 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
14364 for (
unsigned i = 0; i != NumLaneElts; ++i) {
14365 unsigned Index = Imm % NumLaneElts;
14366 Imm /= NumLaneElts;
14367 if (i >= (NumLaneElts / 2))
14369 Indices[l + i] = l + Index;
14373 return Builder.CreateShuffleVector(Ops[0], Ops[1],
14374 ArrayRef(Indices, NumElts),
"shufp");
14376 case X86::BI__builtin_ia32_permdi256:
14377 case X86::BI__builtin_ia32_permdf256:
14378 case X86::BI__builtin_ia32_permdi512:
14379 case X86::BI__builtin_ia32_permdf512: {
14382 unsigned NumElts = Ty->getNumElements();
14386 for (
unsigned l = 0; l != NumElts; l += 4)
14387 for (
unsigned i = 0; i != 4; ++i)
14388 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
14390 return Builder.CreateShuffleVector(Ops[0],
ArrayRef(Indices, NumElts),
14393 case X86::BI__builtin_ia32_palignr128:
14394 case X86::BI__builtin_ia32_palignr256:
14395 case X86::BI__builtin_ia32_palignr512: {
14400 assert(NumElts % 16 == 0);
14404 if (ShiftVal >= 32)
14409 if (ShiftVal > 16) {
14412 Ops[0] = llvm::Constant::getNullValue(Ops[0]->
getType());
14417 for (
unsigned l = 0; l != NumElts; l += 16) {
14418 for (
unsigned i = 0; i != 16; ++i) {
14419 unsigned Idx = ShiftVal + i;
14421 Idx += NumElts - 16;
14422 Indices[l + i] = Idx + l;
14426 return Builder.CreateShuffleVector(Ops[1], Ops[0],
14427 ArrayRef(Indices, NumElts),
"palignr");
14429 case X86::BI__builtin_ia32_alignd128:
14430 case X86::BI__builtin_ia32_alignd256:
14431 case X86::BI__builtin_ia32_alignd512:
14432 case X86::BI__builtin_ia32_alignq128:
14433 case X86::BI__builtin_ia32_alignq256:
14434 case X86::BI__builtin_ia32_alignq512: {
14440 ShiftVal &= NumElts - 1;
14443 for (
unsigned i = 0; i != NumElts; ++i)
14444 Indices[i] = i + ShiftVal;
14446 return Builder.CreateShuffleVector(Ops[1], Ops[0],
14447 ArrayRef(Indices, NumElts),
"valign");
14449 case X86::BI__builtin_ia32_shuf_f32x4_256:
14450 case X86::BI__builtin_ia32_shuf_f64x2_256:
14451 case X86::BI__builtin_ia32_shuf_i32x4_256:
14452 case X86::BI__builtin_ia32_shuf_i64x2_256:
14453 case X86::BI__builtin_ia32_shuf_f32x4:
14454 case X86::BI__builtin_ia32_shuf_f64x2:
14455 case X86::BI__builtin_ia32_shuf_i32x4:
14456 case X86::BI__builtin_ia32_shuf_i64x2: {
14459 unsigned NumElts = Ty->getNumElements();
14460 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
14461 unsigned NumLaneElts = NumElts / NumLanes;
14464 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
14465 unsigned Index = (Imm % NumLanes) * NumLaneElts;
14467 if (l >= (NumElts / 2))
14469 for (
unsigned i = 0; i != NumLaneElts; ++i) {
14470 Indices[l + i] = Index + i;
14474 return Builder.CreateShuffleVector(Ops[0], Ops[1],
14475 ArrayRef(Indices, NumElts),
"shuf");
14478 case X86::BI__builtin_ia32_vperm2f128_pd256:
14479 case X86::BI__builtin_ia32_vperm2f128_ps256:
14480 case X86::BI__builtin_ia32_vperm2f128_si256:
14481 case X86::BI__builtin_ia32_permti256: {
14493 for (
unsigned l = 0; l != 2; ++l) {
14495 if (Imm & (1 << ((l * 4) + 3)))
14496 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->
getType());
14497 else if (Imm & (1 << ((l * 4) + 1)))
14498 OutOps[l] = Ops[1];
14500 OutOps[l] = Ops[0];
14502 for (
unsigned i = 0; i != NumElts/2; ++i) {
14504 unsigned Idx = (l * NumElts) + i;
14507 if (Imm & (1 << (l * 4)))
14509 Indices[(l * (NumElts/2)) + i] = Idx;
14513 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
14514 ArrayRef(Indices, NumElts),
"vperm");
14517 case X86::BI__builtin_ia32_pslldqi128_byteshift:
14518 case X86::BI__builtin_ia32_pslldqi256_byteshift:
14519 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
14523 unsigned NumElts = ResultType->getNumElements() * 8;
14526 if (ShiftVal >= 16)
14527 return llvm::Constant::getNullValue(ResultType);
14531 for (
unsigned l = 0; l != NumElts; l += 16) {
14532 for (
unsigned i = 0; i != 16; ++i) {
14533 unsigned Idx = NumElts + i - ShiftVal;
14534 if (Idx < NumElts) Idx -= NumElts - 16;
14535 Indices[l + i] = Idx + l;
14539 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
14540 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
14541 Value *
Zero = llvm::Constant::getNullValue(VecTy);
14542 Value *SV = Builder.CreateShuffleVector(
14543 Zero, Cast,
ArrayRef(Indices, NumElts),
"pslldq");
14544 return Builder.CreateBitCast(SV, Ops[0]->
getType(),
"cast");
14546 case X86::BI__builtin_ia32_psrldqi128_byteshift:
14547 case X86::BI__builtin_ia32_psrldqi256_byteshift:
14548 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
14552 unsigned NumElts = ResultType->getNumElements() * 8;
14555 if (ShiftVal >= 16)
14556 return llvm::Constant::getNullValue(ResultType);
14560 for (
unsigned l = 0; l != NumElts; l += 16) {
14561 for (
unsigned i = 0; i != 16; ++i) {
14562 unsigned Idx = i + ShiftVal;
14563 if (Idx >= 16) Idx += NumElts - 16;
14564 Indices[l + i] = Idx + l;
14568 auto *VecTy = llvm::FixedVectorType::get(
Int8Ty, NumElts);
14569 Value *
Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
14570 Value *
Zero = llvm::Constant::getNullValue(VecTy);
14571 Value *SV = Builder.CreateShuffleVector(
14572 Cast, Zero,
ArrayRef(Indices, NumElts),
"psrldq");
14573 return Builder.CreateBitCast(SV, ResultType,
"cast");
14575 case X86::BI__builtin_ia32_kshiftliqi:
14576 case X86::BI__builtin_ia32_kshiftlihi:
14577 case X86::BI__builtin_ia32_kshiftlisi:
14578 case X86::BI__builtin_ia32_kshiftlidi: {
14580 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14582 if (ShiftVal >= NumElts)
14583 return llvm::Constant::getNullValue(Ops[0]->
getType());
14588 for (
unsigned i = 0; i != NumElts; ++i)
14589 Indices[i] = NumElts + i - ShiftVal;
14592 Value *SV = Builder.CreateShuffleVector(
14593 Zero, In,
ArrayRef(Indices, NumElts),
"kshiftl");
14594 return Builder.CreateBitCast(SV, Ops[0]->
getType());
14596 case X86::BI__builtin_ia32_kshiftriqi:
14597 case X86::BI__builtin_ia32_kshiftrihi:
14598 case X86::BI__builtin_ia32_kshiftrisi:
14599 case X86::BI__builtin_ia32_kshiftridi: {
14601 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14603 if (ShiftVal >= NumElts)
14604 return llvm::Constant::getNullValue(Ops[0]->
getType());
14609 for (
unsigned i = 0; i != NumElts; ++i)
14610 Indices[i] = i + ShiftVal;
14613 Value *SV = Builder.CreateShuffleVector(
14614 In, Zero,
ArrayRef(Indices, NumElts),
"kshiftr");
14615 return Builder.CreateBitCast(SV, Ops[0]->
getType());
14617 case X86::BI__builtin_ia32_movnti:
14618 case X86::BI__builtin_ia32_movnti64:
14619 case X86::BI__builtin_ia32_movntsd:
14620 case X86::BI__builtin_ia32_movntss: {
14621 llvm::MDNode *
Node = llvm::MDNode::get(
14622 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
14624 Value *Ptr = Ops[0];
14625 Value *Src = Ops[1];
14628 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
14629 BuiltinID == X86::BI__builtin_ia32_movntss)
14630 Src = Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
14633 Value *BC = Builder.CreateBitCast(
14634 Ptr, llvm::PointerType::getUnqual(Src->
getType()),
"cast");
14637 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
14638 SI->setMetadata(llvm::LLVMContext::MD_nontemporal,
Node);
14639 SI->setAlignment(llvm::Align(1));
14643 case X86::BI__builtin_ia32_vprotb:
14644 case X86::BI__builtin_ia32_vprotw:
14645 case X86::BI__builtin_ia32_vprotd:
14646 case X86::BI__builtin_ia32_vprotq:
14647 case X86::BI__builtin_ia32_vprotbi:
14648 case X86::BI__builtin_ia32_vprotwi:
14649 case X86::BI__builtin_ia32_vprotdi:
14650 case X86::BI__builtin_ia32_vprotqi:
14651 case X86::BI__builtin_ia32_prold128:
14652 case X86::BI__builtin_ia32_prold256:
14653 case X86::BI__builtin_ia32_prold512:
14654 case X86::BI__builtin_ia32_prolq128:
14655 case X86::BI__builtin_ia32_prolq256:
14656 case X86::BI__builtin_ia32_prolq512:
14657 case X86::BI__builtin_ia32_prolvd128:
14658 case X86::BI__builtin_ia32_prolvd256:
14659 case X86::BI__builtin_ia32_prolvd512:
14660 case X86::BI__builtin_ia32_prolvq128:
14661 case X86::BI__builtin_ia32_prolvq256:
14662 case X86::BI__builtin_ia32_prolvq512:
14664 case X86::BI__builtin_ia32_prord128:
14665 case X86::BI__builtin_ia32_prord256:
14666 case X86::BI__builtin_ia32_prord512:
14667 case X86::BI__builtin_ia32_prorq128:
14668 case X86::BI__builtin_ia32_prorq256:
14669 case X86::BI__builtin_ia32_prorq512:
14670 case X86::BI__builtin_ia32_prorvd128:
14671 case X86::BI__builtin_ia32_prorvd256:
14672 case X86::BI__builtin_ia32_prorvd512:
14673 case X86::BI__builtin_ia32_prorvq128:
14674 case X86::BI__builtin_ia32_prorvq256:
14675 case X86::BI__builtin_ia32_prorvq512:
14677 case X86::BI__builtin_ia32_selectb_128:
14678 case X86::BI__builtin_ia32_selectb_256:
14679 case X86::BI__builtin_ia32_selectb_512:
14680 case X86::BI__builtin_ia32_selectw_128:
14681 case X86::BI__builtin_ia32_selectw_256:
14682 case X86::BI__builtin_ia32_selectw_512:
14683 case X86::BI__builtin_ia32_selectd_128:
14684 case X86::BI__builtin_ia32_selectd_256:
14685 case X86::BI__builtin_ia32_selectd_512:
14686 case X86::BI__builtin_ia32_selectq_128:
14687 case X86::BI__builtin_ia32_selectq_256:
14688 case X86::BI__builtin_ia32_selectq_512:
14689 case X86::BI__builtin_ia32_selectph_128:
14690 case X86::BI__builtin_ia32_selectph_256:
14691 case X86::BI__builtin_ia32_selectph_512:
14692 case X86::BI__builtin_ia32_selectpbf_128:
14693 case X86::BI__builtin_ia32_selectpbf_256:
14694 case X86::BI__builtin_ia32_selectpbf_512:
14695 case X86::BI__builtin_ia32_selectps_128:
14696 case X86::BI__builtin_ia32_selectps_256:
14697 case X86::BI__builtin_ia32_selectps_512:
14698 case X86::BI__builtin_ia32_selectpd_128:
14699 case X86::BI__builtin_ia32_selectpd_256:
14700 case X86::BI__builtin_ia32_selectpd_512:
14702 case X86::BI__builtin_ia32_selectsh_128:
14703 case X86::BI__builtin_ia32_selectsbf_128:
14704 case X86::BI__builtin_ia32_selectss_128:
14705 case X86::BI__builtin_ia32_selectsd_128: {
14706 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14707 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14709 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
14711 case X86::BI__builtin_ia32_cmpb128_mask:
14712 case X86::BI__builtin_ia32_cmpb256_mask:
14713 case X86::BI__builtin_ia32_cmpb512_mask:
14714 case X86::BI__builtin_ia32_cmpw128_mask:
14715 case X86::BI__builtin_ia32_cmpw256_mask:
14716 case X86::BI__builtin_ia32_cmpw512_mask:
14717 case X86::BI__builtin_ia32_cmpd128_mask:
14718 case X86::BI__builtin_ia32_cmpd256_mask:
14719 case X86::BI__builtin_ia32_cmpd512_mask:
14720 case X86::BI__builtin_ia32_cmpq128_mask:
14721 case X86::BI__builtin_ia32_cmpq256_mask:
14722 case X86::BI__builtin_ia32_cmpq512_mask: {
14726 case X86::BI__builtin_ia32_ucmpb128_mask:
14727 case X86::BI__builtin_ia32_ucmpb256_mask:
14728 case X86::BI__builtin_ia32_ucmpb512_mask:
14729 case X86::BI__builtin_ia32_ucmpw128_mask:
14730 case X86::BI__builtin_ia32_ucmpw256_mask:
14731 case X86::BI__builtin_ia32_ucmpw512_mask:
14732 case X86::BI__builtin_ia32_ucmpd128_mask:
14733 case X86::BI__builtin_ia32_ucmpd256_mask:
14734 case X86::BI__builtin_ia32_ucmpd512_mask:
14735 case X86::BI__builtin_ia32_ucmpq128_mask:
14736 case X86::BI__builtin_ia32_ucmpq256_mask:
14737 case X86::BI__builtin_ia32_ucmpq512_mask: {
14741 case X86::BI__builtin_ia32_vpcomb:
14742 case X86::BI__builtin_ia32_vpcomw:
14743 case X86::BI__builtin_ia32_vpcomd:
14744 case X86::BI__builtin_ia32_vpcomq:
14746 case X86::BI__builtin_ia32_vpcomub:
14747 case X86::BI__builtin_ia32_vpcomuw:
14748 case X86::BI__builtin_ia32_vpcomud:
14749 case X86::BI__builtin_ia32_vpcomuq:
14752 case X86::BI__builtin_ia32_kortestcqi:
14753 case X86::BI__builtin_ia32_kortestchi:
14754 case X86::BI__builtin_ia32_kortestcsi:
14755 case X86::BI__builtin_ia32_kortestcdi: {
14757 Value *
C = llvm::Constant::getAllOnesValue(Ops[0]->
getType());
14758 Value *Cmp = Builder.CreateICmpEQ(Or,
C);
14761 case X86::BI__builtin_ia32_kortestzqi:
14762 case X86::BI__builtin_ia32_kortestzhi:
14763 case X86::BI__builtin_ia32_kortestzsi:
14764 case X86::BI__builtin_ia32_kortestzdi: {
14766 Value *
C = llvm::Constant::getNullValue(Ops[0]->
getType());
14767 Value *Cmp = Builder.CreateICmpEQ(Or,
C);
14771 case X86::BI__builtin_ia32_ktestcqi:
14772 case X86::BI__builtin_ia32_ktestzqi:
14773 case X86::BI__builtin_ia32_ktestchi:
14774 case X86::BI__builtin_ia32_ktestzhi:
14775 case X86::BI__builtin_ia32_ktestcsi:
14776 case X86::BI__builtin_ia32_ktestzsi:
14777 case X86::BI__builtin_ia32_ktestcdi:
14778 case X86::BI__builtin_ia32_ktestzdi: {
14780 switch (BuiltinID) {
14781 default: llvm_unreachable(
"Unsupported intrinsic!");
14782 case X86::BI__builtin_ia32_ktestcqi:
14783 IID = Intrinsic::x86_avx512_ktestc_b;
14785 case X86::BI__builtin_ia32_ktestzqi:
14786 IID = Intrinsic::x86_avx512_ktestz_b;
14788 case X86::BI__builtin_ia32_ktestchi:
14789 IID = Intrinsic::x86_avx512_ktestc_w;
14791 case X86::BI__builtin_ia32_ktestzhi:
14792 IID = Intrinsic::x86_avx512_ktestz_w;
14794 case X86::BI__builtin_ia32_ktestcsi:
14795 IID = Intrinsic::x86_avx512_ktestc_d;
14797 case X86::BI__builtin_ia32_ktestzsi:
14798 IID = Intrinsic::x86_avx512_ktestz_d;
14800 case X86::BI__builtin_ia32_ktestcdi:
14801 IID = Intrinsic::x86_avx512_ktestc_q;
14803 case X86::BI__builtin_ia32_ktestzdi:
14804 IID = Intrinsic::x86_avx512_ktestz_q;
14808 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14812 return Builder.CreateCall(Intr, {LHS, RHS});
14815 case X86::BI__builtin_ia32_kaddqi:
14816 case X86::BI__builtin_ia32_kaddhi:
14817 case X86::BI__builtin_ia32_kaddsi:
14818 case X86::BI__builtin_ia32_kadddi: {
14820 switch (BuiltinID) {
14821 default: llvm_unreachable(
"Unsupported intrinsic!");
14822 case X86::BI__builtin_ia32_kaddqi:
14823 IID = Intrinsic::x86_avx512_kadd_b;
14825 case X86::BI__builtin_ia32_kaddhi:
14826 IID = Intrinsic::x86_avx512_kadd_w;
14828 case X86::BI__builtin_ia32_kaddsi:
14829 IID = Intrinsic::x86_avx512_kadd_d;
14831 case X86::BI__builtin_ia32_kadddi:
14832 IID = Intrinsic::x86_avx512_kadd_q;
14836 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14840 Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
14841 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14843 case X86::BI__builtin_ia32_kandqi:
14844 case X86::BI__builtin_ia32_kandhi:
14845 case X86::BI__builtin_ia32_kandsi:
14846 case X86::BI__builtin_ia32_kanddi:
14848 case X86::BI__builtin_ia32_kandnqi:
14849 case X86::BI__builtin_ia32_kandnhi:
14850 case X86::BI__builtin_ia32_kandnsi:
14851 case X86::BI__builtin_ia32_kandndi:
14853 case X86::BI__builtin_ia32_korqi:
14854 case X86::BI__builtin_ia32_korhi:
14855 case X86::BI__builtin_ia32_korsi:
14856 case X86::BI__builtin_ia32_kordi:
14858 case X86::BI__builtin_ia32_kxnorqi:
14859 case X86::BI__builtin_ia32_kxnorhi:
14860 case X86::BI__builtin_ia32_kxnorsi:
14861 case X86::BI__builtin_ia32_kxnordi:
14863 case X86::BI__builtin_ia32_kxorqi:
14864 case X86::BI__builtin_ia32_kxorhi:
14865 case X86::BI__builtin_ia32_kxorsi:
14866 case X86::BI__builtin_ia32_kxordi:
14868 case X86::BI__builtin_ia32_knotqi:
14869 case X86::BI__builtin_ia32_knothi:
14870 case X86::BI__builtin_ia32_knotsi:
14871 case X86::BI__builtin_ia32_knotdi: {
14872 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14874 return Builder.CreateBitCast(Builder.CreateNot(Res),
14875 Ops[0]->getType());
14877 case X86::BI__builtin_ia32_kmovb:
14878 case X86::BI__builtin_ia32_kmovw:
14879 case X86::BI__builtin_ia32_kmovd:
14880 case X86::BI__builtin_ia32_kmovq: {
14884 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14886 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14889 case X86::BI__builtin_ia32_kunpckdi:
14890 case X86::BI__builtin_ia32_kunpcksi:
14891 case X86::BI__builtin_ia32_kunpckhi: {
14892 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
14896 for (
unsigned i = 0; i != NumElts; ++i)
14901 LHS = Builder.CreateShuffleVector(LHS, LHS,
ArrayRef(Indices, NumElts / 2));
14902 RHS = Builder.CreateShuffleVector(RHS, RHS,
ArrayRef(Indices, NumElts / 2));
14906 Builder.CreateShuffleVector(RHS, LHS,
ArrayRef(Indices, NumElts));
14907 return Builder.CreateBitCast(Res, Ops[0]->
getType());
14910 case X86::BI__builtin_ia32_vplzcntd_128:
14911 case X86::BI__builtin_ia32_vplzcntd_256:
14912 case X86::BI__builtin_ia32_vplzcntd_512:
14913 case X86::BI__builtin_ia32_vplzcntq_128:
14914 case X86::BI__builtin_ia32_vplzcntq_256:
14915 case X86::BI__builtin_ia32_vplzcntq_512: {
14917 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(
false)});
14919 case X86::BI__builtin_ia32_sqrtss:
14920 case X86::BI__builtin_ia32_sqrtsd: {
14921 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
14923 if (Builder.getIsFPConstrained()) {
14924 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14927 A = Builder.CreateConstrainedFPCall(F, {A});
14930 A = Builder.CreateCall(F, {A});
14932 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
14934 case X86::BI__builtin_ia32_sqrtsh_round_mask:
14935 case X86::BI__builtin_ia32_sqrtsd_round_mask:
14936 case X86::BI__builtin_ia32_sqrtss_round_mask: {
14943 switch (BuiltinID) {
14945 llvm_unreachable(
"Unsupported intrinsic!");
14946 case X86::BI__builtin_ia32_sqrtsh_round_mask:
14947 IID = Intrinsic::x86_avx512fp16_mask_sqrt_sh;
14949 case X86::BI__builtin_ia32_sqrtsd_round_mask:
14950 IID = Intrinsic::x86_avx512_mask_sqrt_sd;
14952 case X86::BI__builtin_ia32_sqrtss_round_mask:
14953 IID = Intrinsic::x86_avx512_mask_sqrt_ss;
14958 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
14960 if (Builder.getIsFPConstrained()) {
14961 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
14964 A = Builder.CreateConstrainedFPCall(F, A);
14967 A = Builder.CreateCall(F, A);
14969 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
14971 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
14973 case X86::BI__builtin_ia32_sqrtpd256:
14974 case X86::BI__builtin_ia32_sqrtpd:
14975 case X86::BI__builtin_ia32_sqrtps256:
14976 case X86::BI__builtin_ia32_sqrtps:
14977 case X86::BI__builtin_ia32_sqrtph256:
14978 case X86::BI__builtin_ia32_sqrtph:
14979 case X86::BI__builtin_ia32_sqrtph512:
14980 case X86::BI__builtin_ia32_sqrtps512:
14981 case X86::BI__builtin_ia32_sqrtpd512: {
14982 if (Ops.size() == 2) {
14989 switch (BuiltinID) {
14991 llvm_unreachable(
"Unsupported intrinsic!");
14992 case X86::BI__builtin_ia32_sqrtph512:
14993 IID = Intrinsic::x86_avx512fp16_sqrt_ph_512;
14995 case X86::BI__builtin_ia32_sqrtps512:
14996 IID = Intrinsic::x86_avx512_sqrt_ps_512;
14998 case X86::BI__builtin_ia32_sqrtpd512:
14999 IID = Intrinsic::x86_avx512_sqrt_pd_512;
15005 if (Builder.getIsFPConstrained()) {
15006 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15007 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_sqrt,
15009 return Builder.CreateConstrainedFPCall(F, Ops[0]);
15012 return Builder.CreateCall(F, Ops[0]);
15016 case X86::BI__builtin_ia32_pmuludq128:
15017 case X86::BI__builtin_ia32_pmuludq256:
15018 case X86::BI__builtin_ia32_pmuludq512:
15021 case X86::BI__builtin_ia32_pmuldq128:
15022 case X86::BI__builtin_ia32_pmuldq256:
15023 case X86::BI__builtin_ia32_pmuldq512:
15026 case X86::BI__builtin_ia32_pternlogd512_mask:
15027 case X86::BI__builtin_ia32_pternlogq512_mask:
15028 case X86::BI__builtin_ia32_pternlogd128_mask:
15029 case X86::BI__builtin_ia32_pternlogd256_mask:
15030 case X86::BI__builtin_ia32_pternlogq128_mask:
15031 case X86::BI__builtin_ia32_pternlogq256_mask:
15034 case X86::BI__builtin_ia32_pternlogd512_maskz:
15035 case X86::BI__builtin_ia32_pternlogq512_maskz:
15036 case X86::BI__builtin_ia32_pternlogd128_maskz:
15037 case X86::BI__builtin_ia32_pternlogd256_maskz:
15038 case X86::BI__builtin_ia32_pternlogq128_maskz:
15039 case X86::BI__builtin_ia32_pternlogq256_maskz:
15042 case X86::BI__builtin_ia32_vpshldd128:
15043 case X86::BI__builtin_ia32_vpshldd256:
15044 case X86::BI__builtin_ia32_vpshldd512:
15045 case X86::BI__builtin_ia32_vpshldq128:
15046 case X86::BI__builtin_ia32_vpshldq256:
15047 case X86::BI__builtin_ia32_vpshldq512:
15048 case X86::BI__builtin_ia32_vpshldw128:
15049 case X86::BI__builtin_ia32_vpshldw256:
15050 case X86::BI__builtin_ia32_vpshldw512:
15053 case X86::BI__builtin_ia32_vpshrdd128:
15054 case X86::BI__builtin_ia32_vpshrdd256:
15055 case X86::BI__builtin_ia32_vpshrdd512:
15056 case X86::BI__builtin_ia32_vpshrdq128:
15057 case X86::BI__builtin_ia32_vpshrdq256:
15058 case X86::BI__builtin_ia32_vpshrdq512:
15059 case X86::BI__builtin_ia32_vpshrdw128:
15060 case X86::BI__builtin_ia32_vpshrdw256:
15061 case X86::BI__builtin_ia32_vpshrdw512:
15065 case X86::BI__builtin_ia32_vpshldvd128:
15066 case X86::BI__builtin_ia32_vpshldvd256:
15067 case X86::BI__builtin_ia32_vpshldvd512:
15068 case X86::BI__builtin_ia32_vpshldvq128:
15069 case X86::BI__builtin_ia32_vpshldvq256:
15070 case X86::BI__builtin_ia32_vpshldvq512:
15071 case X86::BI__builtin_ia32_vpshldvw128:
15072 case X86::BI__builtin_ia32_vpshldvw256:
15073 case X86::BI__builtin_ia32_vpshldvw512:
15076 case X86::BI__builtin_ia32_vpshrdvd128:
15077 case X86::BI__builtin_ia32_vpshrdvd256:
15078 case X86::BI__builtin_ia32_vpshrdvd512:
15079 case X86::BI__builtin_ia32_vpshrdvq128:
15080 case X86::BI__builtin_ia32_vpshrdvq256:
15081 case X86::BI__builtin_ia32_vpshrdvq512:
15082 case X86::BI__builtin_ia32_vpshrdvw128:
15083 case X86::BI__builtin_ia32_vpshrdvw256:
15084 case X86::BI__builtin_ia32_vpshrdvw512:
15089 case X86::BI__builtin_ia32_reduce_fadd_pd512:
15090 case X86::BI__builtin_ia32_reduce_fadd_ps512:
15091 case X86::BI__builtin_ia32_reduce_fadd_ph512:
15092 case X86::BI__builtin_ia32_reduce_fadd_ph256:
15093 case X86::BI__builtin_ia32_reduce_fadd_ph128: {
15096 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
15097 Builder.getFastMathFlags().setAllowReassoc();
15098 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15100 case X86::BI__builtin_ia32_reduce_fmul_pd512:
15101 case X86::BI__builtin_ia32_reduce_fmul_ps512:
15102 case X86::BI__builtin_ia32_reduce_fmul_ph512:
15103 case X86::BI__builtin_ia32_reduce_fmul_ph256:
15104 case X86::BI__builtin_ia32_reduce_fmul_ph128: {
15107 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
15108 Builder.getFastMathFlags().setAllowReassoc();
15109 return Builder.CreateCall(F, {Ops[0], Ops[1]});
15111 case X86::BI__builtin_ia32_reduce_fmax_pd512:
15112 case X86::BI__builtin_ia32_reduce_fmax_ps512:
15113 case X86::BI__builtin_ia32_reduce_fmax_ph512:
15114 case X86::BI__builtin_ia32_reduce_fmax_ph256:
15115 case X86::BI__builtin_ia32_reduce_fmax_ph128: {
15118 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
15119 Builder.getFastMathFlags().setNoNaNs();
15120 return Builder.CreateCall(F, {Ops[0]});
15122 case X86::BI__builtin_ia32_reduce_fmin_pd512:
15123 case X86::BI__builtin_ia32_reduce_fmin_ps512:
15124 case X86::BI__builtin_ia32_reduce_fmin_ph512:
15125 case X86::BI__builtin_ia32_reduce_fmin_ph256:
15126 case X86::BI__builtin_ia32_reduce_fmin_ph128: {
15129 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
15130 Builder.getFastMathFlags().setNoNaNs();
15131 return Builder.CreateCall(F, {Ops[0]});
15135 case X86::BI__builtin_ia32_pswapdsf:
15136 case X86::BI__builtin_ia32_pswapdsi: {
15138 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
15140 return Builder.CreateCall(F, Ops,
"pswapd");
15142 case X86::BI__builtin_ia32_rdrand16_step:
15143 case X86::BI__builtin_ia32_rdrand32_step:
15144 case X86::BI__builtin_ia32_rdrand64_step:
15145 case X86::BI__builtin_ia32_rdseed16_step:
15146 case X86::BI__builtin_ia32_rdseed32_step:
15147 case X86::BI__builtin_ia32_rdseed64_step: {
15149 switch (BuiltinID) {
15150 default: llvm_unreachable(
"Unsupported intrinsic!");
15151 case X86::BI__builtin_ia32_rdrand16_step:
15152 ID = Intrinsic::x86_rdrand_16;
15154 case X86::BI__builtin_ia32_rdrand32_step:
15155 ID = Intrinsic::x86_rdrand_32;
15157 case X86::BI__builtin_ia32_rdrand64_step:
15158 ID = Intrinsic::x86_rdrand_64;
15160 case X86::BI__builtin_ia32_rdseed16_step:
15161 ID = Intrinsic::x86_rdseed_16;
15163 case X86::BI__builtin_ia32_rdseed32_step:
15164 ID = Intrinsic::x86_rdseed_32;
15166 case X86::BI__builtin_ia32_rdseed64_step:
15167 ID = Intrinsic::x86_rdseed_64;
15172 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
15174 return Builder.CreateExtractValue(Call, 1);
15176 case X86::BI__builtin_ia32_addcarryx_u32:
15177 case X86::BI__builtin_ia32_addcarryx_u64:
15178 case X86::BI__builtin_ia32_subborrow_u32:
15179 case X86::BI__builtin_ia32_subborrow_u64: {
15181 switch (BuiltinID) {
15182 default: llvm_unreachable(
"Unsupported intrinsic!");
15183 case X86::BI__builtin_ia32_addcarryx_u32:
15184 IID = Intrinsic::x86_addcarry_32;
15186 case X86::BI__builtin_ia32_addcarryx_u64:
15187 IID = Intrinsic::x86_addcarry_64;
15189 case X86::BI__builtin_ia32_subborrow_u32:
15190 IID = Intrinsic::x86_subborrow_32;
15192 case X86::BI__builtin_ia32_subborrow_u64:
15193 IID = Intrinsic::x86_subborrow_64;
15198 { Ops[0], Ops[1], Ops[2] });
15199 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
15201 return Builder.CreateExtractValue(Call, 0);
15204 case X86::BI__builtin_ia32_fpclassps128_mask:
15205 case X86::BI__builtin_ia32_fpclassps256_mask:
15206 case X86::BI__builtin_ia32_fpclassps512_mask:
15207 case X86::BI__builtin_ia32_fpclassph128_mask:
15208 case X86::BI__builtin_ia32_fpclassph256_mask:
15209 case X86::BI__builtin_ia32_fpclassph512_mask:
15210 case X86::BI__builtin_ia32_fpclasspd128_mask:
15211 case X86::BI__builtin_ia32_fpclasspd256_mask:
15212 case X86::BI__builtin_ia32_fpclasspd512_mask: {
15215 Value *MaskIn = Ops[2];
15216 Ops.erase(&Ops[2]);
15219 switch (BuiltinID) {
15220 default: llvm_unreachable(
"Unsupported intrinsic!");
15221 case X86::BI__builtin_ia32_fpclassph128_mask:
15222 ID = Intrinsic::x86_avx512fp16_fpclass_ph_128;
15224 case X86::BI__builtin_ia32_fpclassph256_mask:
15225 ID = Intrinsic::x86_avx512fp16_fpclass_ph_256;
15227 case X86::BI__builtin_ia32_fpclassph512_mask:
15228 ID = Intrinsic::x86_avx512fp16_fpclass_ph_512;
15230 case X86::BI__builtin_ia32_fpclassps128_mask:
15231 ID = Intrinsic::x86_avx512_fpclass_ps_128;
15233 case X86::BI__builtin_ia32_fpclassps256_mask:
15234 ID = Intrinsic::x86_avx512_fpclass_ps_256;
15236 case X86::BI__builtin_ia32_fpclassps512_mask:
15237 ID = Intrinsic::x86_avx512_fpclass_ps_512;
15239 case X86::BI__builtin_ia32_fpclasspd128_mask:
15240 ID = Intrinsic::x86_avx512_fpclass_pd_128;
15242 case X86::BI__builtin_ia32_fpclasspd256_mask:
15243 ID = Intrinsic::x86_avx512_fpclass_pd_256;
15245 case X86::BI__builtin_ia32_fpclasspd512_mask:
15246 ID = Intrinsic::x86_avx512_fpclass_pd_512;
15254 case X86::BI__builtin_ia32_vp2intersect_q_512:
15255 case X86::BI__builtin_ia32_vp2intersect_q_256:
15256 case X86::BI__builtin_ia32_vp2intersect_q_128:
15257 case X86::BI__builtin_ia32_vp2intersect_d_512:
15258 case X86::BI__builtin_ia32_vp2intersect_d_256:
15259 case X86::BI__builtin_ia32_vp2intersect_d_128: {
15264 switch (BuiltinID) {
15265 default: llvm_unreachable(
"Unsupported intrinsic!");
15266 case X86::BI__builtin_ia32_vp2intersect_q_512:
15267 ID = Intrinsic::x86_avx512_vp2intersect_q_512;
15269 case X86::BI__builtin_ia32_vp2intersect_q_256:
15270 ID = Intrinsic::x86_avx512_vp2intersect_q_256;
15272 case X86::BI__builtin_ia32_vp2intersect_q_128:
15273 ID = Intrinsic::x86_avx512_vp2intersect_q_128;
15275 case X86::BI__builtin_ia32_vp2intersect_d_512:
15276 ID = Intrinsic::x86_avx512_vp2intersect_d_512;
15278 case X86::BI__builtin_ia32_vp2intersect_d_256:
15279 ID = Intrinsic::x86_avx512_vp2intersect_d_256;
15281 case X86::BI__builtin_ia32_vp2intersect_d_128:
15282 ID = Intrinsic::x86_avx512_vp2intersect_d_128;
15287 Value *
Result = Builder.CreateExtractValue(Call, 0);
15289 Builder.CreateDefaultAlignedStore(
Result, Ops[2]);
15291 Result = Builder.CreateExtractValue(Call, 1);
15293 return Builder.CreateDefaultAlignedStore(
Result, Ops[3]);
15296 case X86::BI__builtin_ia32_vpmultishiftqb128:
15297 case X86::BI__builtin_ia32_vpmultishiftqb256:
15298 case X86::BI__builtin_ia32_vpmultishiftqb512: {
15300 switch (BuiltinID) {
15301 default: llvm_unreachable(
"Unsupported intrinsic!");
15302 case X86::BI__builtin_ia32_vpmultishiftqb128:
15303 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
15305 case X86::BI__builtin_ia32_vpmultishiftqb256:
15306 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
15308 case X86::BI__builtin_ia32_vpmultishiftqb512:
15309 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
15316 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
15317 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
15318 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
15321 Value *MaskIn = Ops[2];
15322 Ops.erase(&Ops[2]);
15325 switch (BuiltinID) {
15326 default: llvm_unreachable(
"Unsupported intrinsic!");
15327 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
15328 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
15330 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
15331 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
15333 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
15334 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
15343 case X86::BI__builtin_ia32_cmpeqps:
15344 case X86::BI__builtin_ia32_cmpeqpd:
15345 return getVectorFCmpIR(CmpInst::FCMP_OEQ,
false);
15346 case X86::BI__builtin_ia32_cmpltps:
15347 case X86::BI__builtin_ia32_cmpltpd:
15348 return getVectorFCmpIR(CmpInst::FCMP_OLT,
true);
15349 case X86::BI__builtin_ia32_cmpleps:
15350 case X86::BI__builtin_ia32_cmplepd:
15351 return getVectorFCmpIR(CmpInst::FCMP_OLE,
true);
15352 case X86::BI__builtin_ia32_cmpunordps:
15353 case X86::BI__builtin_ia32_cmpunordpd:
15354 return getVectorFCmpIR(CmpInst::FCMP_UNO,
false);
15355 case X86::BI__builtin_ia32_cmpneqps:
15356 case X86::BI__builtin_ia32_cmpneqpd:
15357 return getVectorFCmpIR(CmpInst::FCMP_UNE,
false);
15358 case X86::BI__builtin_ia32_cmpnltps:
15359 case X86::BI__builtin_ia32_cmpnltpd:
15360 return getVectorFCmpIR(CmpInst::FCMP_UGE,
true);
15361 case X86::BI__builtin_ia32_cmpnleps:
15362 case X86::BI__builtin_ia32_cmpnlepd:
15363 return getVectorFCmpIR(CmpInst::FCMP_UGT,
true);
15364 case X86::BI__builtin_ia32_cmpordps:
15365 case X86::BI__builtin_ia32_cmpordpd:
15366 return getVectorFCmpIR(CmpInst::FCMP_ORD,
false);
15367 case X86::BI__builtin_ia32_cmpph128_mask:
15368 case X86::BI__builtin_ia32_cmpph256_mask:
15369 case X86::BI__builtin_ia32_cmpph512_mask:
15370 case X86::BI__builtin_ia32_cmpps128_mask:
15371 case X86::BI__builtin_ia32_cmpps256_mask:
15372 case X86::BI__builtin_ia32_cmpps512_mask:
15373 case X86::BI__builtin_ia32_cmppd128_mask:
15374 case X86::BI__builtin_ia32_cmppd256_mask:
15375 case X86::BI__builtin_ia32_cmppd512_mask:
15378 case X86::BI__builtin_ia32_cmpps:
15379 case X86::BI__builtin_ia32_cmpps256:
15380 case X86::BI__builtin_ia32_cmppd:
15381 case X86::BI__builtin_ia32_cmppd256: {
15394 FCmpInst::Predicate Pred;
15398 switch (CC & 0xf) {
15399 case 0x00: Pred = FCmpInst::FCMP_OEQ; IsSignaling =
false;
break;
15400 case 0x01: Pred = FCmpInst::FCMP_OLT; IsSignaling =
true;
break;
15401 case 0x02: Pred = FCmpInst::FCMP_OLE; IsSignaling =
true;
break;
15402 case 0x03: Pred = FCmpInst::FCMP_UNO; IsSignaling =
false;
break;
15403 case 0x04: Pred = FCmpInst::FCMP_UNE; IsSignaling =
false;
break;
15404 case 0x05: Pred = FCmpInst::FCMP_UGE; IsSignaling =
true;
break;
15405 case 0x06: Pred = FCmpInst::FCMP_UGT; IsSignaling =
true;
break;
15406 case 0x07: Pred = FCmpInst::FCMP_ORD; IsSignaling =
false;
break;
15407 case 0x08: Pred = FCmpInst::FCMP_UEQ; IsSignaling =
false;
break;
15408 case 0x09: Pred = FCmpInst::FCMP_ULT; IsSignaling =
true;
break;
15409 case 0x0a: Pred = FCmpInst::FCMP_ULE; IsSignaling =
true;
break;
15410 case 0x0b: Pred = FCmpInst::FCMP_FALSE; IsSignaling =
false;
break;
15411 case 0x0c: Pred = FCmpInst::FCMP_ONE; IsSignaling =
false;
break;
15412 case 0x0d: Pred = FCmpInst::FCMP_OGE; IsSignaling =
true;
break;
15413 case 0x0e: Pred = FCmpInst::FCMP_OGT; IsSignaling =
true;
break;
15414 case 0x0f: Pred = FCmpInst::FCMP_TRUE; IsSignaling =
false;
break;
15415 default: llvm_unreachable(
"Unhandled CC");
15420 IsSignaling = !IsSignaling;
15427 if (Builder.getIsFPConstrained() &&
15428 (Pred == FCmpInst::FCMP_TRUE || Pred == FCmpInst::FCMP_FALSE ||
15432 switch (BuiltinID) {
15433 default: llvm_unreachable(
"Unexpected builtin");
15434 case X86::BI__builtin_ia32_cmpps:
15435 IID = Intrinsic::x86_sse_cmp_ps;
15437 case X86::BI__builtin_ia32_cmpps256:
15438 IID = Intrinsic::x86_avx_cmp_ps_256;
15440 case X86::BI__builtin_ia32_cmppd:
15441 IID = Intrinsic::x86_sse2_cmp_pd;
15443 case X86::BI__builtin_ia32_cmppd256:
15444 IID = Intrinsic::x86_avx_cmp_pd_256;
15446 case X86::BI__builtin_ia32_cmpps512_mask:
15447 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
15449 case X86::BI__builtin_ia32_cmppd512_mask:
15450 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
15452 case X86::BI__builtin_ia32_cmpps128_mask:
15453 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
15455 case X86::BI__builtin_ia32_cmpps256_mask:
15456 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
15458 case X86::BI__builtin_ia32_cmppd128_mask:
15459 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
15461 case X86::BI__builtin_ia32_cmppd256_mask:
15462 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
15471 Value *Cmp = Builder.CreateCall(Intr, Ops);
15475 return Builder.CreateCall(Intr, Ops);
15489 Cmp = Builder.CreateFCmpS(Pred, Ops[0], Ops[1]);
15491 Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
15495 return getVectorFCmpIR(Pred, IsSignaling);
15499 case X86::BI__builtin_ia32_cmpeqss:
15500 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
15501 case X86::BI__builtin_ia32_cmpltss:
15502 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
15503 case X86::BI__builtin_ia32_cmpless:
15504 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
15505 case X86::BI__builtin_ia32_cmpunordss:
15506 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
15507 case X86::BI__builtin_ia32_cmpneqss:
15508 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
15509 case X86::BI__builtin_ia32_cmpnltss:
15510 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
15511 case X86::BI__builtin_ia32_cmpnless:
15512 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
15513 case X86::BI__builtin_ia32_cmpordss:
15514 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
15515 case X86::BI__builtin_ia32_cmpeqsd:
15516 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
15517 case X86::BI__builtin_ia32_cmpltsd:
15518 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
15519 case X86::BI__builtin_ia32_cmplesd:
15520 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
15521 case X86::BI__builtin_ia32_cmpunordsd:
15522 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
15523 case X86::BI__builtin_ia32_cmpneqsd:
15524 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
15525 case X86::BI__builtin_ia32_cmpnltsd:
15526 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
15527 case X86::BI__builtin_ia32_cmpnlesd:
15528 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
15529 case X86::BI__builtin_ia32_cmpordsd:
15530 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
15533 case X86::BI__builtin_ia32_vcvtph2ps:
15534 case X86::BI__builtin_ia32_vcvtph2ps256:
15535 case X86::BI__builtin_ia32_vcvtph2ps_mask:
15536 case X86::BI__builtin_ia32_vcvtph2ps256_mask:
15537 case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
15538 CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*
this, E);
15543 case X86::BI__builtin_ia32_cvtneps2bf16_128_mask: {
15546 cast<llvm::FixedVectorType>(Ops[0]->
getType())->getNumElements());
15547 Intrinsic::ID IID = Intrinsic::x86_avx512bf16_mask_cvtneps2bf16_128;
15550 case X86::BI__builtin_ia32_cvtsbf162ss_32:
15551 return Builder.CreateFPExt(Ops[0], Builder.getFloatTy());
15553 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
15554 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask: {
15556 switch (BuiltinID) {
15557 default: llvm_unreachable(
"Unsupported intrinsic!");
15558 case X86::BI__builtin_ia32_cvtneps2bf16_256_mask:
15559 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_256;
15561 case X86::BI__builtin_ia32_cvtneps2bf16_512_mask:
15562 IID = Intrinsic::x86_avx512bf16_cvtneps2bf16_512;
15569 case X86::BI__cpuid:
15570 case X86::BI__cpuidex: {
15572 Value *SubFuncId = BuiltinID == X86::BI__cpuidex
15576 llvm::StructType *CpuidRetTy =
15578 llvm::FunctionType *FTy =
15581 StringRef
Asm, Constraints;
15582 if (
getTarget().getTriple().getArch() == llvm::Triple::x86) {
15584 Constraints =
"={ax},={bx},={cx},={dx},{ax},{cx}";
15587 Asm =
"xchgq %rbx, ${1:q}\n"
15589 "xchgq %rbx, ${1:q}";
15590 Constraints =
"={ax},=r,={cx},={dx},0,2";
15593 llvm::InlineAsm *IA = llvm::InlineAsm::get(FTy,
Asm, Constraints,
15595 Value *IACall = Builder.CreateCall(IA, {FuncId, SubFuncId});
15598 for (
unsigned i = 0; i < 4; i++) {
15599 Value *Extracted = Builder.CreateExtractValue(IACall, i);
15600 Value *StorePtr = Builder.CreateConstInBoundsGEP1_32(
Int32Ty, BasePtr, i);
15609 case X86::BI__emul:
15610 case X86::BI__emulu: {
15612 bool isSigned = (BuiltinID == X86::BI__emul);
15613 Value *LHS = Builder.CreateIntCast(Ops[0],
Int64Ty, isSigned);
15614 Value *RHS = Builder.CreateIntCast(Ops[1],
Int64Ty, isSigned);
15615 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
15617 case X86::BI__mulh:
15618 case X86::BI__umulh:
15619 case X86::BI_mul128:
15620 case X86::BI_umul128: {
15622 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
15624 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
15625 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
15626 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
15628 Value *MulResult, *HigherBits;
15630 MulResult = Builder.CreateNSWMul(LHS, RHS);
15631 HigherBits = Builder.CreateAShr(MulResult, 64);
15633 MulResult = Builder.CreateNUWMul(LHS, RHS);
15634 HigherBits = Builder.CreateLShr(MulResult, 64);
15636 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
15638 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
15642 Builder.CreateStore(HigherBits, HighBitsAddress);
15643 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
15646 case X86::BI__faststorefence: {
15647 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
15648 llvm::SyncScope::System);
15650 case X86::BI__shiftleft128:
15651 case X86::BI__shiftright128: {
15653 BuiltinID == X86::BI__shiftleft128 ? Intrinsic::fshl : Intrinsic::fshr,
15658 std::swap(Ops[0], Ops[1]);
15659 Ops[2] = Builder.CreateZExt(Ops[2],
Int64Ty);
15660 return Builder.CreateCall(F, Ops);
15662 case X86::BI_ReadWriteBarrier:
15663 case X86::BI_ReadBarrier:
15664 case X86::BI_WriteBarrier: {
15665 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
15666 llvm::SyncScope::SingleThread);
15669 case X86::BI_AddressOfReturnAddress: {
15672 return Builder.CreateCall(F);
15674 case X86::BI__stosb: {
15677 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], Align(1),
true);
15682 case X86::BI__int2c: {
15684 llvm::FunctionType *FTy = llvm::FunctionType::get(
VoidTy,
false);
15685 llvm::InlineAsm *IA =
15686 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
15687 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
15689 llvm::Attribute::NoReturn);
15690 llvm::CallInst *CI = Builder.CreateCall(IA);
15691 CI->setAttributes(NoReturnAttr);
15694 case X86::BI__readfsbyte:
15695 case X86::BI__readfsword:
15696 case X86::BI__readfsdword:
15697 case X86::BI__readfsqword: {
15700 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(
IntTy, 257));
15701 LoadInst *
Load = Builder.CreateAlignedLoad(
15703 Load->setVolatile(
true);
15706 case X86::BI__readgsbyte:
15707 case X86::BI__readgsword:
15708 case X86::BI__readgsdword:
15709 case X86::BI__readgsqword: {
15712 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(
IntTy, 256));
15713 LoadInst *
Load = Builder.CreateAlignedLoad(
15715 Load->setVolatile(
true);
15718 case X86::BI__builtin_ia32_encodekey128_u32: {
15719 Intrinsic::ID IID = Intrinsic::x86_encodekey128;
15723 for (
int i = 0; i < 3; ++i) {
15724 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15725 Value *Ptr = Builder.CreateConstGEP1_32(
Int8Ty, Ops[2], i * 16);
15726 Ptr = Builder.CreateBitCast(
15727 Ptr, llvm::PointerType::getUnqual(Extract->
getType()));
15728 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
15731 return Builder.CreateExtractValue(Call, 0);
15733 case X86::BI__builtin_ia32_encodekey256_u32: {
15734 Intrinsic::ID IID = Intrinsic::x86_encodekey256;
15739 for (
int i = 0; i < 4; ++i) {
15740 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15741 Value *Ptr = Builder.CreateConstGEP1_32(
Int8Ty, Ops[3], i * 16);
15742 Ptr = Builder.CreateBitCast(
15743 Ptr, llvm::PointerType::getUnqual(Extract->
getType()));
15744 Builder.CreateAlignedStore(Extract, Ptr, Align(1));
15747 return Builder.CreateExtractValue(Call, 0);
15749 case X86::BI__builtin_ia32_aesenc128kl_u8:
15750 case X86::BI__builtin_ia32_aesdec128kl_u8:
15751 case X86::BI__builtin_ia32_aesenc256kl_u8:
15752 case X86::BI__builtin_ia32_aesdec256kl_u8: {
15754 StringRef BlockName;
15755 switch (BuiltinID) {
15757 llvm_unreachable(
"Unexpected builtin");
15758 case X86::BI__builtin_ia32_aesenc128kl_u8:
15759 IID = Intrinsic::x86_aesenc128kl;
15760 BlockName =
"aesenc128kl";
15762 case X86::BI__builtin_ia32_aesdec128kl_u8:
15763 IID = Intrinsic::x86_aesdec128kl;
15764 BlockName =
"aesdec128kl";
15766 case X86::BI__builtin_ia32_aesenc256kl_u8:
15767 IID = Intrinsic::x86_aesenc256kl;
15768 BlockName =
"aesenc256kl";
15770 case X86::BI__builtin_ia32_aesdec256kl_u8:
15771 IID = Intrinsic::x86_aesdec256kl;
15772 BlockName =
"aesdec256kl";
15778 BasicBlock *NoError =
15783 Value *
Ret = Builder.CreateExtractValue(Call, 0);
15784 Value *Succ = Builder.CreateTrunc(Ret, Builder.getInt1Ty());
15785 Value *Out = Builder.CreateExtractValue(Call, 1);
15786 Builder.CreateCondBr(Succ, NoError,
Error);
15788 Builder.SetInsertPoint(NoError);
15789 Builder.CreateDefaultAlignedStore(Out, Ops[0]);
15790 Builder.CreateBr(End);
15792 Builder.SetInsertPoint(
Error);
15793 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
15794 Builder.CreateDefaultAlignedStore(Zero, Ops[0]);
15795 Builder.CreateBr(End);
15797 Builder.SetInsertPoint(End);
15798 return Builder.CreateExtractValue(Call, 0);
15800 case X86::BI__builtin_ia32_aesencwide128kl_u8:
15801 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
15802 case X86::BI__builtin_ia32_aesencwide256kl_u8:
15803 case X86::BI__builtin_ia32_aesdecwide256kl_u8: {
15805 StringRef BlockName;
15806 switch (BuiltinID) {
15807 case X86::BI__builtin_ia32_aesencwide128kl_u8:
15808 IID = Intrinsic::x86_aesencwide128kl;
15809 BlockName =
"aesencwide128kl";
15811 case X86::BI__builtin_ia32_aesdecwide128kl_u8:
15812 IID = Intrinsic::x86_aesdecwide128kl;
15813 BlockName =
"aesdecwide128kl";
15815 case X86::BI__builtin_ia32_aesencwide256kl_u8:
15816 IID = Intrinsic::x86_aesencwide256kl;
15817 BlockName =
"aesencwide256kl";
15819 case X86::BI__builtin_ia32_aesdecwide256kl_u8:
15820 IID = Intrinsic::x86_aesdecwide256kl;
15821 BlockName =
"aesdecwide256kl";
15825 llvm::Type *Ty = FixedVectorType::get(Builder.getInt64Ty(), 2);
15828 for (
int i = 0; i != 8; ++i) {
15829 Value *Ptr = Builder.CreateConstGEP1_32(Ty, Ops[1], i);
15830 InOps[i + 1] = Builder.CreateAlignedLoad(Ty, Ptr, Align(16));
15835 BasicBlock *NoError =
15840 Value *
Ret = Builder.CreateExtractValue(Call, 0);
15841 Value *Succ = Builder.CreateTrunc(Ret, Builder.getInt1Ty());
15842 Builder.CreateCondBr(Succ, NoError,
Error);
15844 Builder.SetInsertPoint(NoError);
15845 for (
int i = 0; i != 8; ++i) {
15846 Value *Extract = Builder.CreateExtractValue(Call, i + 1);
15847 Value *Ptr = Builder.CreateConstGEP1_32(Extract->
getType(), Ops[0], i);
15848 Builder.CreateAlignedStore(Extract, Ptr, Align(16));
15850 Builder.CreateBr(End);
15852 Builder.SetInsertPoint(
Error);
15853 for (
int i = 0; i != 8; ++i) {
15854 Value *Out = Builder.CreateExtractValue(Call, i + 1);
15855 Constant *
Zero = llvm::Constant::getNullValue(Out->getType());
15856 Value *Ptr = Builder.CreateConstGEP1_32(Out->getType(), Ops[0], i);
15857 Builder.CreateAlignedStore(Zero, Ptr, Align(16));
15859 Builder.CreateBr(End);
15861 Builder.SetInsertPoint(End);
15862 return Builder.CreateExtractValue(Call, 0);
15864 case X86::BI__builtin_ia32_vfcmaddcph512_mask:
15867 case X86::BI__builtin_ia32_vfmaddcph512_mask: {
15868 Intrinsic::ID IID = IsConjFMA
15869 ? Intrinsic::x86_avx512fp16_mask_vfcmadd_cph_512
15870 : Intrinsic::x86_avx512fp16_mask_vfmadd_cph_512;
15874 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask:
15877 case X86::BI__builtin_ia32_vfmaddcsh_round_mask: {
15878 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
15879 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
15881 Value *
And = Builder.CreateAnd(Ops[3], llvm::ConstantInt::get(
Int8Ty, 1));
15884 case X86::BI__builtin_ia32_vfcmaddcsh_round_mask3:
15887 case X86::BI__builtin_ia32_vfmaddcsh_round_mask3: {
15888 Intrinsic::ID IID = IsConjFMA ? Intrinsic::x86_avx512fp16_mask_vfcmadd_csh
15889 : Intrinsic::x86_avx512fp16_mask_vfmadd_csh;
15891 static constexpr int Mask[] = {0, 5, 6, 7};
15892 return Builder.CreateShuffleVector(Call, Ops[2], Mask);
15894 case X86::BI__builtin_ia32_prefetchi:
15895 return Builder.CreateCall(
15897 {Ops[0], llvm::ConstantInt::get(Int32Ty, 0), Ops[1],
15898 llvm::ConstantInt::get(Int32Ty, 0)});
15916 Intrinsic::ID ID = Intrinsic::not_intrinsic;
15918 switch (BuiltinID) {
15919 default:
return nullptr;
15923 case PPC::BI__builtin_ppc_get_timebase:
15924 return Builder.CreateCall(
CGM.
getIntrinsic(Intrinsic::readcyclecounter));
15927 case PPC::BI__builtin_altivec_lvx:
15928 case PPC::BI__builtin_altivec_lvxl:
15929 case PPC::BI__builtin_altivec_lvebx:
15930 case PPC::BI__builtin_altivec_lvehx:
15931 case PPC::BI__builtin_altivec_lvewx:
15932 case PPC::BI__builtin_altivec_lvsl:
15933 case PPC::BI__builtin_altivec_lvsr:
15934 case PPC::BI__builtin_vsx_lxvd2x:
15935 case PPC::BI__builtin_vsx_lxvw4x:
15936 case PPC::BI__builtin_vsx_lxvd2x_be:
15937 case PPC::BI__builtin_vsx_lxvw4x_be:
15938 case PPC::BI__builtin_vsx_lxvl:
15939 case PPC::BI__builtin_vsx_lxvll:
15944 if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
15945 BuiltinID == PPC::BI__builtin_vsx_lxvll){
15946 Ops[0] = Builder.CreateBitCast(Ops[0],
Int8PtrTy);
15948 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
15949 Ops[0] = Builder.CreateGEP(
Int8Ty, Ops[1], Ops[0]);
15953 switch (BuiltinID) {
15954 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
15955 case PPC::BI__builtin_altivec_lvx:
15956 ID = Intrinsic::ppc_altivec_lvx;
15958 case PPC::BI__builtin_altivec_lvxl:
15959 ID = Intrinsic::ppc_altivec_lvxl;
15961 case PPC::BI__builtin_altivec_lvebx:
15962 ID = Intrinsic::ppc_altivec_lvebx;
15964 case PPC::BI__builtin_altivec_lvehx:
15965 ID = Intrinsic::ppc_altivec_lvehx;
15967 case PPC::BI__builtin_altivec_lvewx:
15968 ID = Intrinsic::ppc_altivec_lvewx;
15970 case PPC::BI__builtin_altivec_lvsl:
15971 ID = Intrinsic::ppc_altivec_lvsl;
15973 case PPC::BI__builtin_altivec_lvsr:
15974 ID = Intrinsic::ppc_altivec_lvsr;
15976 case PPC::BI__builtin_vsx_lxvd2x:
15977 ID = Intrinsic::ppc_vsx_lxvd2x;
15979 case PPC::BI__builtin_vsx_lxvw4x:
15980 ID = Intrinsic::ppc_vsx_lxvw4x;
15982 case PPC::BI__builtin_vsx_lxvd2x_be:
15983 ID = Intrinsic::ppc_vsx_lxvd2x_be;
15985 case PPC::BI__builtin_vsx_lxvw4x_be:
15986 ID = Intrinsic::ppc_vsx_lxvw4x_be;
15988 case PPC::BI__builtin_vsx_lxvl:
15989 ID = Intrinsic::ppc_vsx_lxvl;
15991 case PPC::BI__builtin_vsx_lxvll:
15992 ID = Intrinsic::ppc_vsx_lxvll;
15996 return Builder.CreateCall(F, Ops,
"");
16000 case PPC::BI__builtin_altivec_stvx:
16001 case PPC::BI__builtin_altivec_stvxl:
16002 case PPC::BI__builtin_altivec_stvebx:
16003 case PPC::BI__builtin_altivec_stvehx:
16004 case PPC::BI__builtin_altivec_stvewx:
16005 case PPC::BI__builtin_vsx_stxvd2x:
16006 case PPC::BI__builtin_vsx_stxvw4x:
16007 case PPC::BI__builtin_vsx_stxvd2x_be:
16008 case PPC::BI__builtin_vsx_stxvw4x_be:
16009 case PPC::BI__builtin_vsx_stxvl:
16010 case PPC::BI__builtin_vsx_stxvll:
16016 if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
16017 BuiltinID == PPC::BI__builtin_vsx_stxvll ){
16018 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
16020 Ops[2] = Builder.CreateBitCast(Ops[2],
Int8PtrTy);
16021 Ops[1] = Builder.CreateGEP(
Int8Ty, Ops[2], Ops[1]);
16025 switch (BuiltinID) {
16026 default: llvm_unreachable(
"Unsupported st intrinsic!");
16027 case PPC::BI__builtin_altivec_stvx:
16028 ID = Intrinsic::ppc_altivec_stvx;
16030 case PPC::BI__builtin_altivec_stvxl:
16031 ID = Intrinsic::ppc_altivec_stvxl;
16033 case PPC::BI__builtin_altivec_stvebx:
16034 ID = Intrinsic::ppc_altivec_stvebx;
16036 case PPC::BI__builtin_altivec_stvehx:
16037 ID = Intrinsic::ppc_altivec_stvehx;
16039 case PPC::BI__builtin_altivec_stvewx:
16040 ID = Intrinsic::ppc_altivec_stvewx;
16042 case PPC::BI__builtin_vsx_stxvd2x:
16043 ID = Intrinsic::ppc_vsx_stxvd2x;
16045 case PPC::BI__builtin_vsx_stxvw4x:
16046 ID = Intrinsic::ppc_vsx_stxvw4x;
16048 case PPC::BI__builtin_vsx_stxvd2x_be:
16049 ID = Intrinsic::ppc_vsx_stxvd2x_be;
16051 case PPC::BI__builtin_vsx_stxvw4x_be:
16052 ID = Intrinsic::ppc_vsx_stxvw4x_be;
16054 case PPC::BI__builtin_vsx_stxvl:
16055 ID = Intrinsic::ppc_vsx_stxvl;
16057 case PPC::BI__builtin_vsx_stxvll:
16058 ID = Intrinsic::ppc_vsx_stxvll;
16062 return Builder.CreateCall(F, Ops,
"");
16064 case PPC::BI__builtin_vsx_ldrmb: {
16075 if (NumBytes == 16) {
16083 for (
int Idx = 0; Idx < 16; Idx++)
16084 RevMask.push_back(15 - Idx);
16085 return Builder.CreateShuffleVector(LD, LD, RevMask);
16089 llvm::Function *Lvs =
CGM.
getIntrinsic(IsLE ? Intrinsic::ppc_altivec_lvsr
16090 : Intrinsic::ppc_altivec_lvsl);
16091 llvm::Function *Vperm =
CGM.
getIntrinsic(Intrinsic::ppc_altivec_vperm);
16092 Value *HiMem = Builder.CreateGEP(
16093 Int8Ty, Op0, ConstantInt::get(Op1->
getType(), NumBytes - 1));
16094 Value *LoLd = Builder.CreateCall(Lvx, Op0,
"ld.lo");
16095 Value *HiLd = Builder.CreateCall(Lvx, HiMem,
"ld.hi");
16096 Value *Mask1 = Builder.CreateCall(Lvs, Op0,
"mask1");
16098 Op0 = IsLE ? HiLd : LoLd;
16099 Op1 = IsLE ? LoLd : HiLd;
16100 Value *AllElts = Builder.CreateCall(Vperm, {Op0, Op1, Mask1},
"shuffle1");
16101 Constant *
Zero = llvm::Constant::getNullValue(IsLE ? ResTy : AllElts->
getType());
16105 for (
int Idx = 0; Idx < 16; Idx++) {
16106 int Val = (NumBytes - Idx - 1 >= 0) ? (NumBytes - Idx - 1)
16107 : 16 - (NumBytes - Idx);
16108 Consts.push_back(Val);
16110 return Builder.CreateShuffleVector(Builder.CreateBitCast(AllElts, ResTy),
16114 for (
int Idx = 0; Idx < 16; Idx++)
16115 Consts.push_back(Builder.getInt8(NumBytes + Idx));
16116 Value *Mask2 = ConstantVector::get(Consts);
16117 return Builder.CreateBitCast(
16118 Builder.CreateCall(Vperm, {Zero, AllElts, Mask2},
"shuffle2"), ResTy);
16120 case PPC::BI__builtin_vsx_strmb: {
16126 auto StoreSubVec = [&](
unsigned Width,
unsigned Offset,
unsigned EltNo) {
16130 Value *StVec = Op2;
16133 for (
int Idx = 0; Idx < 16; Idx++)
16134 RevMask.push_back(15 - Idx);
16135 StVec = Builder.CreateShuffleVector(Op2, Op2, RevMask);
16137 return Builder.CreateStore(
16141 unsigned NumElts = 0;
16144 llvm_unreachable(
"width for stores must be a power of 2");
16162 Value *Vec = Builder.CreateBitCast(
16163 Op2, llvm::FixedVectorType::get(ConvTy, NumElts));
16166 Value *Elt = Builder.CreateExtractElement(Vec, EltNo);
16167 if (IsLE && Width > 1) {
16169 Elt = Builder.CreateCall(F, Elt);
16171 return Builder.CreateStore(
16174 unsigned Stored = 0;
16175 unsigned RemainingBytes = NumBytes;
16177 if (NumBytes == 16)
16178 return StoreSubVec(16, 0, 0);
16179 if (NumBytes >= 8) {
16180 Result = StoreSubVec(8, NumBytes - 8, IsLE ? 0 : 1);
16181 RemainingBytes -= 8;
16184 if (RemainingBytes >= 4) {
16185 Result = StoreSubVec(4, NumBytes - Stored - 4,
16186 IsLE ? (Stored >> 2) : 3 - (Stored >> 2));
16187 RemainingBytes -= 4;
16190 if (RemainingBytes >= 2) {
16191 Result = StoreSubVec(2, NumBytes - Stored - 2,
16192 IsLE ? (Stored >> 1) : 7 - (Stored >> 1));
16193 RemainingBytes -= 2;
16196 if (RemainingBytes)
16198 StoreSubVec(1, NumBytes - Stored - 1, IsLE ? Stored : 15 - Stored);
16202 case PPC::BI__builtin_vsx_xvsqrtsp:
16203 case PPC::BI__builtin_vsx_xvsqrtdp: {
16206 if (Builder.getIsFPConstrained()) {
16208 Intrinsic::experimental_constrained_sqrt, ResultType);
16209 return Builder.CreateConstrainedFPCall(F,
X);
16212 return Builder.CreateCall(F,
X);
16216 case PPC::BI__builtin_altivec_vclzb:
16217 case PPC::BI__builtin_altivec_vclzh:
16218 case PPC::BI__builtin_altivec_vclzw:
16219 case PPC::BI__builtin_altivec_vclzd: {
16222 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
16224 return Builder.CreateCall(F, {
X, Undef});
16226 case PPC::BI__builtin_altivec_vctzb:
16227 case PPC::BI__builtin_altivec_vctzh:
16228 case PPC::BI__builtin_altivec_vctzw:
16229 case PPC::BI__builtin_altivec_vctzd: {
16232 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
16234 return Builder.CreateCall(F, {
X, Undef});
16236 case PPC::BI__builtin_altivec_vinsd:
16237 case PPC::BI__builtin_altivec_vinsw:
16238 case PPC::BI__builtin_altivec_vinsd_elt:
16239 case PPC::BI__builtin_altivec_vinsw_elt: {
16245 bool IsUnaligned = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
16246 BuiltinID == PPC::BI__builtin_altivec_vinsd);
16248 bool Is32bit = (BuiltinID == PPC::BI__builtin_altivec_vinsw ||
16249 BuiltinID == PPC::BI__builtin_altivec_vinsw_elt);
16252 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16254 "Third Arg to vinsw/vinsd intrinsic must be a constant integer!");
16258 int ValidMaxValue = 0;
16260 ValidMaxValue = (Is32bit) ? 12 : 8;
16262 ValidMaxValue = (Is32bit) ? 3 : 1;
16265 int64_t ConstArg = ArgCI->getSExtValue();
16268 std::string RangeErrMsg = IsUnaligned ?
"byte" :
"element";
16269 RangeErrMsg +=
" number " + llvm::to_string(ConstArg);
16270 RangeErrMsg +=
" is outside of the valid range [0, ";
16271 RangeErrMsg += llvm::to_string(ValidMaxValue) +
"]";
16274 if (ConstArg < 0 || ConstArg > ValidMaxValue)
16278 if (!IsUnaligned) {
16279 ConstArg *= Is32bit ? 4 : 8;
16282 ConstArg = (Is32bit ? 12 : 8) - ConstArg;
16285 ID = Is32bit ? Intrinsic::ppc_altivec_vinsw : Intrinsic::ppc_altivec_vinsd;
16286 Op2 = ConstantInt::getSigned(
Int32Ty, ConstArg);
16290 ? Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4))
16291 : Builder.CreateBitCast(Op0,
16292 llvm::FixedVectorType::get(
Int64Ty, 2));
16293 return Builder.CreateBitCast(
16294 Builder.CreateCall(
CGM.
getIntrinsic(ID), {Op0, Op1, Op2}), ResultType);
16296 case PPC::BI__builtin_altivec_vpopcntb:
16297 case PPC::BI__builtin_altivec_vpopcnth:
16298 case PPC::BI__builtin_altivec_vpopcntw:
16299 case PPC::BI__builtin_altivec_vpopcntd: {
16303 return Builder.CreateCall(F,
X);
16305 case PPC::BI__builtin_altivec_vadduqm:
16306 case PPC::BI__builtin_altivec_vsubuqm: {
16309 llvm::Type *Int128Ty = llvm::IntegerType::get(
getLLVMContext(), 128);
16310 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(Int128Ty, 1));
16311 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(Int128Ty, 1));
16312 if (BuiltinID == PPC::BI__builtin_altivec_vadduqm)
16313 return Builder.CreateAdd(Op0, Op1,
"vadduqm");
16315 return Builder.CreateSub(Op0, Op1,
"vsubuqm");
16317 case PPC::BI__builtin_altivec_vaddcuq_c:
16318 case PPC::BI__builtin_altivec_vsubcuq_c: {
16322 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
16324 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
16325 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
16326 ID = (BuiltinID == PPC::BI__builtin_altivec_vaddcuq_c)
16327 ? Intrinsic::ppc_altivec_vaddcuq
16328 : Intrinsic::ppc_altivec_vsubcuq;
16331 case PPC::BI__builtin_altivec_vaddeuqm_c:
16332 case PPC::BI__builtin_altivec_vaddecuq_c:
16333 case PPC::BI__builtin_altivec_vsubeuqm_c:
16334 case PPC::BI__builtin_altivec_vsubecuq_c: {
16339 llvm::Type *V1I128Ty = llvm::FixedVectorType::get(
16341 Ops.push_back(Builder.CreateBitCast(Op0, V1I128Ty));
16342 Ops.push_back(Builder.CreateBitCast(Op1, V1I128Ty));
16343 Ops.push_back(Builder.CreateBitCast(Op2, V1I128Ty));
16344 switch (BuiltinID) {
16346 llvm_unreachable(
"Unsupported intrinsic!");
16347 case PPC::BI__builtin_altivec_vaddeuqm_c:
16348 ID = Intrinsic::ppc_altivec_vaddeuqm;
16350 case PPC::BI__builtin_altivec_vaddecuq_c:
16351 ID = Intrinsic::ppc_altivec_vaddecuq;
16353 case PPC::BI__builtin_altivec_vsubeuqm_c:
16354 ID = Intrinsic::ppc_altivec_vsubeuqm;
16356 case PPC::BI__builtin_altivec_vsubecuq_c:
16357 ID = Intrinsic::ppc_altivec_vsubecuq;
16367 case PPC::BI__builtin_ppc_rldimi:
16368 case PPC::BI__builtin_ppc_rlwimi: {
16373 llvm::Type *Ty = Op0->
getType();
16375 if (BuiltinID == PPC::BI__builtin_ppc_rldimi)
16376 Op2 = Builder.CreateZExt(Op2,
Int64Ty);
16377 Value *
Shift = Builder.CreateCall(F, {Op0, Op0, Op2});
16378 Value *
X = Builder.CreateAnd(Shift, Op3);
16379 Value *Y = Builder.CreateAnd(Op1, Builder.CreateNot(Op3));
16380 return Builder.CreateOr(
X, Y);
16385 case PPC::BI__builtin_ppc_rlwnm: {
16389 llvm::Type *Ty = Op0->
getType();
16391 Value *
Shift = Builder.CreateCall(F, {Op0, Op0, Op1});
16392 return Builder.CreateAnd(Shift, Op2);
16394 case PPC::BI__builtin_ppc_poppar4:
16395 case PPC::BI__builtin_ppc_poppar8: {
16397 llvm::Type *ArgType = Op0->
getType();
16399 Value *Tmp = Builder.CreateCall(F, Op0);
16402 Value *
Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
16403 if (
Result->getType() != ResultType)
16404 Result = Builder.CreateIntCast(
Result, ResultType,
true,
16408 case PPC::BI__builtin_ppc_cmpb: {
16411 if (
getTarget().getTriple().isPPC64()) {
16414 return Builder.CreateCall(F, {Op0, Op1},
"cmpb");
16434 Constant *ShiftAmt = ConstantInt::get(
Int64Ty, 32);
16436 Builder.CreateTrunc(Builder.CreateLShr(Op0, ShiftAmt),
Int32Ty);
16438 Builder.CreateTrunc(Builder.CreateLShr(Op1, ShiftAmt),
Int32Ty);
16439 Value *ResLo = Builder.CreateZExt(
16440 Builder.CreateCall(F, {ArgOneLo, ArgTwoLo},
"cmpb"),
Int64Ty);
16441 Value *ResHiShift = Builder.CreateZExt(
16442 Builder.CreateCall(F, {ArgOneHi, ArgTwoHi},
"cmpb"),
Int64Ty);
16443 Value *ResHi = Builder.CreateShl(ResHiShift, ShiftAmt);
16444 return Builder.CreateOr(ResLo, ResHi);
16447 case PPC::BI__builtin_vsx_xvcpsgnsp:
16448 case PPC::BI__builtin_vsx_xvcpsgndp: {
16452 ID = Intrinsic::copysign;
16454 return Builder.CreateCall(F, {
X, Y});
16457 case PPC::BI__builtin_vsx_xvrspip:
16458 case PPC::BI__builtin_vsx_xvrdpip:
16459 case PPC::BI__builtin_vsx_xvrdpim:
16460 case PPC::BI__builtin_vsx_xvrspim:
16461 case PPC::BI__builtin_vsx_xvrdpi:
16462 case PPC::BI__builtin_vsx_xvrspi:
16463 case PPC::BI__builtin_vsx_xvrdpic:
16464 case PPC::BI__builtin_vsx_xvrspic:
16465 case PPC::BI__builtin_vsx_xvrdpiz:
16466 case PPC::BI__builtin_vsx_xvrspiz: {
16469 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
16470 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
16471 ID = Builder.getIsFPConstrained()
16472 ? Intrinsic::experimental_constrained_floor
16473 : Intrinsic::floor;
16474 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
16475 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
16476 ID = Builder.getIsFPConstrained()
16477 ? Intrinsic::experimental_constrained_round
16478 : Intrinsic::round;
16479 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
16480 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
16481 ID = Builder.getIsFPConstrained()
16482 ? Intrinsic::experimental_constrained_rint
16484 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
16485 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
16486 ID = Builder.getIsFPConstrained()
16487 ? Intrinsic::experimental_constrained_ceil
16489 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
16490 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
16491 ID = Builder.getIsFPConstrained()
16492 ? Intrinsic::experimental_constrained_trunc
16493 : Intrinsic::trunc;
16495 return Builder.getIsFPConstrained() ? Builder.CreateConstrainedFPCall(F,
X)
16496 : Builder.CreateCall(F,
X);
16500 case PPC::BI__builtin_vsx_xvabsdp:
16501 case PPC::BI__builtin_vsx_xvabssp: {
16505 return Builder.CreateCall(F,
X);
16509 case PPC::BI__builtin_ppc_recipdivf:
16510 case PPC::BI__builtin_ppc_recipdivd:
16511 case PPC::BI__builtin_ppc_rsqrtf:
16512 case PPC::BI__builtin_ppc_rsqrtd: {
16513 FastMathFlags FMF = Builder.getFastMathFlags();
16514 Builder.getFastMathFlags().setFast();
16518 if (BuiltinID == PPC::BI__builtin_ppc_recipdivf ||
16519 BuiltinID == PPC::BI__builtin_ppc_recipdivd) {
16521 Value *FDiv = Builder.CreateFDiv(
X, Y,
"recipdiv");
16522 Builder.getFastMathFlags() &= (FMF);
16525 auto *One = ConstantFP::get(ResultType, 1.0);
16527 Value *FDiv = Builder.CreateFDiv(One, Builder.CreateCall(F,
X),
"rsqrt");
16528 Builder.getFastMathFlags() &= (FMF);
16531 case PPC::BI__builtin_ppc_alignx: {
16535 if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
16536 AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
16537 llvm::Value::MaximumAlignment);
16541 AlignmentCI,
nullptr);
16544 case PPC::BI__builtin_ppc_rdlam: {
16548 llvm::Type *Ty = Op0->
getType();
16549 Value *ShiftAmt = Builder.CreateIntCast(Op1, Ty,
false);
16551 Value *Rotate = Builder.CreateCall(F, {Op0, Op0, ShiftAmt});
16552 return Builder.CreateAnd(Rotate, Op2);
16554 case PPC::BI__builtin_ppc_load2r: {
16557 Value *LoadIntrinsic = Builder.CreateCall(F, {Op0});
16558 return Builder.CreateTrunc(LoadIntrinsic,
Int16Ty);
16561 case PPC::BI__builtin_ppc_fnmsub:
16562 case PPC::BI__builtin_ppc_fnmsubs:
16563 case PPC::BI__builtin_vsx_xvmaddadp:
16564 case PPC::BI__builtin_vsx_xvmaddasp:
16565 case PPC::BI__builtin_vsx_xvnmaddadp:
16566 case PPC::BI__builtin_vsx_xvnmaddasp:
16567 case PPC::BI__builtin_vsx_xvmsubadp:
16568 case PPC::BI__builtin_vsx_xvmsubasp:
16569 case PPC::BI__builtin_vsx_xvnmsubadp:
16570 case PPC::BI__builtin_vsx_xvnmsubasp: {
16576 if (Builder.getIsFPConstrained())
16577 F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
16580 switch (BuiltinID) {
16581 case PPC::BI__builtin_vsx_xvmaddadp:
16582 case PPC::BI__builtin_vsx_xvmaddasp:
16583 if (Builder.getIsFPConstrained())
16584 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
16586 return Builder.CreateCall(F, {
X, Y, Z});
16587 case PPC::BI__builtin_vsx_xvnmaddadp:
16588 case PPC::BI__builtin_vsx_xvnmaddasp:
16589 if (Builder.getIsFPConstrained())
16590 return Builder.CreateFNeg(
16591 Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
16593 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
16594 case PPC::BI__builtin_vsx_xvmsubadp:
16595 case PPC::BI__builtin_vsx_xvmsubasp:
16596 if (Builder.getIsFPConstrained())
16597 return Builder.CreateConstrainedFPCall(
16598 F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
16600 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
16601 case PPC::BI__builtin_ppc_fnmsub:
16602 case PPC::BI__builtin_ppc_fnmsubs:
16603 case PPC::BI__builtin_vsx_xvnmsubadp:
16604 case PPC::BI__builtin_vsx_xvnmsubasp:
16605 if (Builder.getIsFPConstrained())
16606 return Builder.CreateFNeg(
16607 Builder.CreateConstrainedFPCall(
16608 F, {X, Y, Builder.CreateFNeg(Z,
"neg")}),
16611 return Builder.CreateCall(
16614 llvm_unreachable(
"Unknown FMA operation");
16618 case PPC::BI__builtin_vsx_insertword: {
16626 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16628 "Third arg to xxinsertw intrinsic must be constant integer");
16629 const int64_t MaxIndex = 12;
16630 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
16637 std::swap(Op0, Op1);
16641 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
16645 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
16646 Op0 = Builder.CreateShuffleVector(Op0, Op0,
ArrayRef<int>{1, 0});
16649 Index = MaxIndex - Index;
16653 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
16654 Op2 = ConstantInt::getSigned(
Int32Ty, Index);
16655 return Builder.CreateCall(F, {Op0, Op1, Op2});
16658 case PPC::BI__builtin_vsx_extractuword: {
16661 llvm::Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
16664 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
16668 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op1);
16670 "Second Arg to xxextractuw intrinsic must be a constant integer!");
16671 const int64_t MaxIndex = 12;
16672 int64_t Index = std::clamp(ArgCI->getSExtValue(), (int64_t)0, MaxIndex);
16676 Index = MaxIndex - Index;
16677 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
16680 Value *
Call = Builder.CreateCall(F, {Op0, Op1});
16682 Value *ShuffleCall =
16683 Builder.CreateShuffleVector(Call, Call,
ArrayRef<int>{1, 0});
16684 return ShuffleCall;
16686 Op1 = ConstantInt::getSigned(
Int32Ty, Index);
16687 return Builder.CreateCall(F, {Op0, Op1});
16691 case PPC::BI__builtin_vsx_xxpermdi: {
16695 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16696 assert(ArgCI &&
"Third arg must be constant integer!");
16698 unsigned Index = ArgCI->getZExtValue();
16699 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int64Ty, 2));
16700 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int64Ty, 2));
16705 int ElemIdx0 = (Index & 2) >> 1;
16706 int ElemIdx1 = 2 + (Index & 1);
16708 int ShuffleElts[2] = {ElemIdx0, ElemIdx1};
16709 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
16712 return Builder.CreateBitCast(ShuffleCall, RetTy);
16715 case PPC::BI__builtin_vsx_xxsldwi: {
16719 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Op2);
16720 assert(ArgCI &&
"Third argument must be a compile time constant");
16721 unsigned Index = ArgCI->getZExtValue() & 0x3;
16722 Op0 = Builder.CreateBitCast(Op0, llvm::FixedVectorType::get(
Int32Ty, 4));
16723 Op1 = Builder.CreateBitCast(Op1, llvm::FixedVectorType::get(
Int32Ty, 4));
16734 ElemIdx0 = (8 - Index) % 8;
16735 ElemIdx1 = (9 - Index) % 8;
16736 ElemIdx2 = (10 - Index) % 8;
16737 ElemIdx3 = (11 - Index) % 8;
16741 ElemIdx1 = Index + 1;
16742 ElemIdx2 = Index + 2;
16743 ElemIdx3 = Index + 3;
16746 int ShuffleElts[4] = {ElemIdx0, ElemIdx1, ElemIdx2, ElemIdx3};
16747 Value *ShuffleCall = Builder.CreateShuffleVector(Op0, Op1, ShuffleElts);
16750 return Builder.CreateBitCast(ShuffleCall, RetTy);
16753 case PPC::BI__builtin_pack_vector_int128: {
16757 Value *PoisonValue =
16758 llvm::PoisonValue::get(llvm::FixedVectorType::get(Op0->
getType(), 2));
16759 Value *Res = Builder.CreateInsertElement(
16760 PoisonValue, Op0, (uint64_t)(isLittleEndian ? 1 : 0));
16761 Res = Builder.CreateInsertElement(Res, Op1,
16762 (uint64_t)(isLittleEndian ? 0 : 1));
16766 case PPC::BI__builtin_unpack_vector_int128: {
16770 Value *Unpacked = Builder.CreateBitCast(
16774 Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue());
16776 return Builder.CreateExtractElement(Unpacked, Index);
16779 case PPC::BI__builtin_ppc_sthcx: {
16783 return Builder.CreateCall(F, {Op0, Op1});
16792#define CUSTOM_BUILTIN(Name, Intr, Types, Accumulate, Feature) \
16793 case PPC::BI__builtin_##Name:
16794#include "clang/Basic/BuiltinsPPC.def"
16797 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
16806 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc ||
16807 BuiltinID == PPC::BI__builtin_vsx_disassemble_pair ||
16808 BuiltinID == PPC::BI__builtin_mma_disassemble_pair) {
16809 unsigned NumVecs = 2;
16810 auto Intrinsic = Intrinsic::ppc_vsx_disassemble_pair;
16811 if (BuiltinID == PPC::BI__builtin_mma_disassemble_acc) {
16813 Intrinsic = Intrinsic::ppc_mma_disassemble_acc;
16817 Value *Vec = Builder.CreateLoad(Addr);
16818 Value *
Call = Builder.CreateCall(F, {Vec});
16819 llvm::Type *VTy = llvm::FixedVectorType::get(
Int8Ty, 16);
16820 Value *Ptr = Ops[0];
16821 for (
unsigned i=0; i<NumVecs; i++) {
16822 Value *Vec = Builder.CreateExtractValue(Call, i);
16823 llvm::ConstantInt* Index = llvm::ConstantInt::get(
IntTy, i);
16824 Value *GEP = Builder.CreateInBoundsGEP(VTy, Ptr, Index);
16825 Builder.CreateAlignedStore(Vec, GEP, MaybeAlign(16));
16829 if (BuiltinID == PPC::BI__builtin_vsx_build_pair ||
16830 BuiltinID == PPC::BI__builtin_mma_build_acc) {
16838 std::reverse(Ops.begin() + 1, Ops.end());
16841 switch (BuiltinID) {
16842 #define CUSTOM_BUILTIN(Name, Intr, Types, Acc, Feature) \
16843 case PPC::BI__builtin_##Name: \
16844 ID = Intrinsic::ppc_##Intr; \
16845 Accumulate = Acc; \
16847 #include "clang/Basic/BuiltinsPPC.def"
16849 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
16850 BuiltinID == PPC::BI__builtin_vsx_stxvp ||
16851 BuiltinID == PPC::BI__builtin_mma_lxvp ||
16852 BuiltinID == PPC::BI__builtin_mma_stxvp) {
16853 if (BuiltinID == PPC::BI__builtin_vsx_lxvp ||
16854 BuiltinID == PPC::BI__builtin_mma_lxvp) {
16855 Ops[1] = Builder.CreateBitCast(Ops[1],
Int8PtrTy);
16856 Ops[0] = Builder.CreateGEP(
Int8Ty, Ops[1], Ops[0]);
16858 Ops[2] = Builder.CreateBitCast(Ops[2],
Int8PtrTy);
16859 Ops[1] = Builder.CreateGEP(
Int8Ty, Ops[2], Ops[1]);
16863 return Builder.CreateCall(F, Ops,
"");
16868 Value *Acc = Builder.CreateLoad(Addr);
16869 CallOps.push_back(Acc);
16871 for (
unsigned i=1; i<Ops.size(); i++)
16872 CallOps.push_back(Ops[i]);
16874 Value *
Call = Builder.CreateCall(F, CallOps);
16875 return Builder.CreateAlignedStore(Call, Ops[0], MaybeAlign(64));
16878 case PPC::BI__builtin_ppc_compare_and_swap:
16879 case PPC::BI__builtin_ppc_compare_and_swaplp: {
16882 Value *OldVal = Builder.CreateLoad(OldValAddr);
16888 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Monotonic,
true);
16896 Value *LoadedVal = Pair.first.getScalarVal();
16897 Builder.CreateStore(LoadedVal, OldValAddr);
16898 return Builder.CreateZExt(Pair.second, Builder.getInt32Ty());
16900 case PPC::BI__builtin_ppc_fetch_and_add:
16901 case PPC::BI__builtin_ppc_fetch_and_addlp: {
16903 llvm::AtomicOrdering::Monotonic);
16905 case PPC::BI__builtin_ppc_fetch_and_and:
16906 case PPC::BI__builtin_ppc_fetch_and_andlp: {
16908 llvm::AtomicOrdering::Monotonic);
16911 case PPC::BI__builtin_ppc_fetch_and_or:
16912 case PPC::BI__builtin_ppc_fetch_and_orlp: {
16914 llvm::AtomicOrdering::Monotonic);
16916 case PPC::BI__builtin_ppc_fetch_and_swap:
16917 case PPC::BI__builtin_ppc_fetch_and_swaplp: {
16919 llvm::AtomicOrdering::Monotonic);
16921 case PPC::BI__builtin_ppc_ldarx:
16922 case PPC::BI__builtin_ppc_lwarx:
16923 case PPC::BI__builtin_ppc_lharx:
16924 case PPC::BI__builtin_ppc_lbarx:
16926 case PPC::BI__builtin_ppc_mfspr: {
16932 return Builder.CreateCall(F, {Op0});
16934 case PPC::BI__builtin_ppc_mtspr: {
16941 return Builder.CreateCall(F, {Op0, Op1});
16943 case PPC::BI__builtin_ppc_popcntb: {
16945 llvm::Type *ArgType = ArgValue->
getType();
16946 Function *F =
CGM.
getIntrinsic(Intrinsic::ppc_popcntb, {ArgType, ArgType});
16947 return Builder.CreateCall(F, {ArgValue},
"popcntb");
16949 case PPC::BI__builtin_ppc_mtfsf: {
16956 return Builder.CreateCall(F, {Op0,
Cast},
"");
16959 case PPC::BI__builtin_ppc_swdiv_nochk:
16960 case PPC::BI__builtin_ppc_swdivs_nochk: {
16963 FastMathFlags FMF = Builder.getFastMathFlags();
16964 Builder.getFastMathFlags().setFast();
16965 Value *FDiv = Builder.CreateFDiv(Op0, Op1,
"swdiv_nochk");
16966 Builder.getFastMathFlags() &= (FMF);
16969 case PPC::BI__builtin_ppc_fric:
16971 *
this, E, Intrinsic::rint,
16972 Intrinsic::experimental_constrained_rint))
16974 case PPC::BI__builtin_ppc_frim:
16975 case PPC::BI__builtin_ppc_frims:
16977 *
this, E, Intrinsic::floor,
16978 Intrinsic::experimental_constrained_floor))
16980 case PPC::BI__builtin_ppc_frin:
16981 case PPC::BI__builtin_ppc_frins:
16983 *
this, E, Intrinsic::round,
16984 Intrinsic::experimental_constrained_round))
16986 case PPC::BI__builtin_ppc_frip:
16987 case PPC::BI__builtin_ppc_frips:
16989 *
this, E, Intrinsic::ceil,
16990 Intrinsic::experimental_constrained_ceil))
16992 case PPC::BI__builtin_ppc_friz:
16993 case PPC::BI__builtin_ppc_frizs:
16995 *
this, E, Intrinsic::trunc,
16996 Intrinsic::experimental_constrained_trunc))
16998 case PPC::BI__builtin_ppc_fsqrt:
16999 case PPC::BI__builtin_ppc_fsqrts:
17001 *
this, E, Intrinsic::sqrt,
17002 Intrinsic::experimental_constrained_sqrt))
17004 case PPC::BI__builtin_ppc_test_data_class: {
17007 return Builder.CreateCall(
17009 {Op0, Op1},
"test_data_class");
17011 case PPC::BI__builtin_ppc_maxfe: {
17017 {Op0, Op1, Op2, Op3});
17019 case PPC::BI__builtin_ppc_maxfl: {
17025 {Op0, Op1, Op2, Op3});
17027 case PPC::BI__builtin_ppc_maxfs: {
17033 {Op0, Op1, Op2, Op3});
17035 case PPC::BI__builtin_ppc_minfe: {
17041 {Op0, Op1, Op2, Op3});
17043 case PPC::BI__builtin_ppc_minfl: {
17049 {Op0, Op1, Op2, Op3});
17051 case PPC::BI__builtin_ppc_minfs: {
17057 {Op0, Op1, Op2, Op3});
17059 case PPC::BI__builtin_ppc_swdiv:
17060 case PPC::BI__builtin_ppc_swdivs: {
17063 return Builder.CreateFDiv(Op0, Op1,
"swdiv");
17076 Attribute::getWithDereferenceableBytes(
Call->getContext(), 64));
17077 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(4)));
17082 if (RetTy ==
Call->getType())
17091 Attribute::getWithDereferenceableBytes(
Call->getContext(), 256));
17092 Call->addRetAttr(Attribute::getWithAlignment(
Call->getContext(), Align(8)));
17104 Offset = llvm::ConstantInt::get(CGF.
Int32Ty, 12 + Index * 2);
17105 DP = EmitAMDGPUImplicitArgPtr(CGF);
17108 Offset = llvm::ConstantInt::get(CGF.
Int32Ty, 4 + Index * 2);
17109 DP = EmitAMDGPUDispatchPtr(CGF);
17116 llvm::MDNode *RNode = MDHelper.createRange(APInt(16, 1),
17118 LD->setMetadata(llvm::LLVMContext::MD_range, RNode);
17119 LD->setMetadata(llvm::LLVMContext::MD_noundef,
17121 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
17128 const unsigned XOffset = 12;
17129 auto *DP = EmitAMDGPUDispatchPtr(CGF);
17131 auto *
Offset = llvm::ConstantInt::get(CGF.
Int32Ty, XOffset + Index * 4);
17135 LD->setMetadata(llvm::LLVMContext::MD_invariant_load,
17148 llvm::AtomicOrdering &AO,
17149 llvm::SyncScope::ID &SSID) {
17153 assert(llvm::isValidAtomicOrderingCABI(ord));
17154 switch (
static_cast<llvm::AtomicOrderingCABI
>(ord)) {
17155 case llvm::AtomicOrderingCABI::acquire:
17156 case llvm::AtomicOrderingCABI::consume:
17157 AO = llvm::AtomicOrdering::Acquire;
17159 case llvm::AtomicOrderingCABI::release:
17160 AO = llvm::AtomicOrdering::Release;
17162 case llvm::AtomicOrderingCABI::acq_rel:
17163 AO = llvm::AtomicOrdering::AcquireRelease;
17165 case llvm::AtomicOrderingCABI::seq_cst:
17166 AO = llvm::AtomicOrdering::SequentiallyConsistent;
17168 case llvm::AtomicOrderingCABI::relaxed:
17169 AO = llvm::AtomicOrdering::Monotonic;
17174 llvm::getConstantStringInfo(
Scope, scp);
17180 llvm::AtomicOrdering AO = llvm::AtomicOrdering::SequentiallyConsistent;
17181 llvm::SyncScope::ID SSID;
17182 switch (BuiltinID) {
17183 case AMDGPU::BI__builtin_amdgcn_div_scale:
17184 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
17197 llvm::Value *Tmp = Builder.CreateCall(Callee, {
X, Y, Z});
17199 llvm::Value *
Result = Builder.CreateExtractValue(Tmp, 0);
17200 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
17204 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
17205 Builder.CreateStore(FlagExt, FlagOutPtr);
17208 case AMDGPU::BI__builtin_amdgcn_div_fmas:
17209 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
17217 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
17218 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
17221 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
17223 case AMDGPU::BI__builtin_amdgcn_mov_dpp8:
17225 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
17226 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
17228 for (
unsigned I = 0; I != E->
getNumArgs(); ++I)
17230 assert(Args.size() == 5 || Args.size() == 6);
17231 if (Args.size() == 5)
17232 Args.insert(Args.begin(), llvm::PoisonValue::get(Args[0]->getType()));
17235 return Builder.CreateCall(F, Args);
17237 case AMDGPU::BI__builtin_amdgcn_div_fixup:
17238 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
17239 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
17241 case AMDGPU::BI__builtin_amdgcn_trig_preop:
17242 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
17244 case AMDGPU::BI__builtin_amdgcn_rcp:
17245 case AMDGPU::BI__builtin_amdgcn_rcpf:
17246 case AMDGPU::BI__builtin_amdgcn_rcph:
17248 case AMDGPU::BI__builtin_amdgcn_sqrt:
17249 case AMDGPU::BI__builtin_amdgcn_sqrtf:
17250 case AMDGPU::BI__builtin_amdgcn_sqrth:
17252 case AMDGPU::BI__builtin_amdgcn_rsq:
17253 case AMDGPU::BI__builtin_amdgcn_rsqf:
17254 case AMDGPU::BI__builtin_amdgcn_rsqh:
17256 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
17257 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
17259 case AMDGPU::BI__builtin_amdgcn_sinf:
17260 case AMDGPU::BI__builtin_amdgcn_sinh:
17262 case AMDGPU::BI__builtin_amdgcn_cosf:
17263 case AMDGPU::BI__builtin_amdgcn_cosh:
17265 case AMDGPU::BI__builtin_amdgcn_dispatch_ptr:
17266 return EmitAMDGPUDispatchPtr(*
this, E);
17267 case AMDGPU::BI__builtin_amdgcn_logf:
17269 case AMDGPU::BI__builtin_amdgcn_exp2f:
17271 case AMDGPU::BI__builtin_amdgcn_log_clampf:
17273 case AMDGPU::BI__builtin_amdgcn_ldexp:
17274 case AMDGPU::BI__builtin_amdgcn_ldexpf:
17275 case AMDGPU::BI__builtin_amdgcn_ldexph: {
17278 llvm::Function *F =
17279 CGM.
getIntrinsic(Intrinsic::ldexp, {Src0->getType(), Src1->getType()});
17280 return Builder.CreateCall(F, {Src0, Src1});
17282 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
17283 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
17284 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
17286 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
17287 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
17290 { Builder.getInt32Ty(), Src0->
getType() });
17291 return Builder.CreateCall(F, Src0);
17293 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
17296 { Builder.getInt16Ty(), Src0->
getType() });
17297 return Builder.CreateCall(F, Src0);
17299 case AMDGPU::BI__builtin_amdgcn_fract:
17300 case AMDGPU::BI__builtin_amdgcn_fractf:
17301 case AMDGPU::BI__builtin_amdgcn_fracth:
17303 case AMDGPU::BI__builtin_amdgcn_lerp:
17305 case AMDGPU::BI__builtin_amdgcn_ubfe:
17307 case AMDGPU::BI__builtin_amdgcn_sbfe:
17309 case AMDGPU::BI__builtin_amdgcn_ballot_w32:
17310 case AMDGPU::BI__builtin_amdgcn_ballot_w64: {
17313 Function *F =
CGM.
getIntrinsic(Intrinsic::amdgcn_ballot, { ResultType });
17314 return Builder.CreateCall(F, { Src });
17316 case AMDGPU::BI__builtin_amdgcn_uicmp:
17317 case AMDGPU::BI__builtin_amdgcn_uicmpl:
17318 case AMDGPU::BI__builtin_amdgcn_sicmp:
17319 case AMDGPU::BI__builtin_amdgcn_sicmpl: {
17326 { Builder.getInt64Ty(), Src0->getType() });
17327 return Builder.CreateCall(F, { Src0, Src1, Src2 });
17329 case AMDGPU::BI__builtin_amdgcn_fcmp:
17330 case AMDGPU::BI__builtin_amdgcn_fcmpf: {
17337 { Builder.getInt64Ty(), Src0->getType() });
17338 return Builder.CreateCall(F, { Src0, Src1, Src2 });
17340 case AMDGPU::BI__builtin_amdgcn_class:
17341 case AMDGPU::BI__builtin_amdgcn_classf:
17342 case AMDGPU::BI__builtin_amdgcn_classh:
17344 case AMDGPU::BI__builtin_amdgcn_fmed3f:
17345 case AMDGPU::BI__builtin_amdgcn_fmed3h:
17347 case AMDGPU::BI__builtin_amdgcn_ds_append:
17348 case AMDGPU::BI__builtin_amdgcn_ds_consume: {
17349 Intrinsic::ID Intrin = BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_append ?
17350 Intrinsic::amdgcn_ds_append : Intrinsic::amdgcn_ds_consume;
17353 return Builder.CreateCall(F, { Src0, Builder.getFalse() });
17355 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
17356 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
17357 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
17358 Intrinsic::ID Intrin;
17359 switch (BuiltinID) {
17360 case AMDGPU::BI__builtin_amdgcn_ds_faddf:
17361 Intrin = Intrinsic::amdgcn_ds_fadd;
17363 case AMDGPU::BI__builtin_amdgcn_ds_fminf:
17364 Intrin = Intrinsic::amdgcn_ds_fmin;
17366 case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
17367 Intrin = Intrinsic::amdgcn_ds_fmax;
17376 llvm::FunctionType *FTy = F->getFunctionType();
17377 llvm::Type *PTy = FTy->getParamType(0);
17378 Src0 = Builder.CreatePointerBitCastOrAddrSpaceCast(Src0, PTy);
17379 return Builder.CreateCall(F, { Src0, Src1, Src2, Src3, Src4 });
17381 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
17382 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
17383 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
17384 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
17385 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
17386 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
17387 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
17388 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
17389 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
17390 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16: {
17393 switch (BuiltinID) {
17394 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
17396 IID = Intrinsic::amdgcn_global_atomic_fadd;
17398 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
17399 ArgTy = llvm::FixedVectorType::get(
17401 IID = Intrinsic::amdgcn_global_atomic_fadd;
17403 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
17404 IID = Intrinsic::amdgcn_global_atomic_fadd;
17406 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
17407 IID = Intrinsic::amdgcn_global_atomic_fmin;
17409 case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
17410 IID = Intrinsic::amdgcn_global_atomic_fmax;
17412 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f64:
17413 IID = Intrinsic::amdgcn_flat_atomic_fadd;
17415 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmin_f64:
17416 IID = Intrinsic::amdgcn_flat_atomic_fmin;
17418 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fmax_f64:
17419 IID = Intrinsic::amdgcn_flat_atomic_fmax;
17421 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_f32:
17423 IID = Intrinsic::amdgcn_flat_atomic_fadd;
17425 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2f16:
17426 ArgTy = llvm::FixedVectorType::get(
17428 IID = Intrinsic::amdgcn_flat_atomic_fadd;
17433 llvm::Function *F =
17435 return Builder.CreateCall(F, {Addr, Val});
17437 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
17438 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16: {
17440 switch (BuiltinID) {
17441 case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2bf16:
17442 IID = Intrinsic::amdgcn_global_atomic_fadd_v2bf16;
17444 case AMDGPU::BI__builtin_amdgcn_flat_atomic_fadd_v2bf16:
17445 IID = Intrinsic::amdgcn_flat_atomic_fadd_v2bf16;
17451 return Builder.CreateCall(F, {Addr, Val});
17453 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
17454 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
17455 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16: {
17458 switch (BuiltinID) {
17459 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
17461 IID = Intrinsic::amdgcn_ds_fadd;
17463 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f64:
17465 IID = Intrinsic::amdgcn_ds_fadd;
17467 case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
17468 ArgTy = llvm::FixedVectorType::get(
17470 IID = Intrinsic::amdgcn_ds_fadd;
17475 llvm::Constant *ZeroI32 = llvm::ConstantInt::getIntegerValue(
17477 llvm::Constant *ZeroI1 = llvm::ConstantInt::getIntegerValue(
17480 return Builder.CreateCall(F, {Addr, Val, ZeroI32, ZeroI32, ZeroI1});
17482 case AMDGPU::BI__builtin_amdgcn_read_exec: {
17485 CI->setConvergent();
17488 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
17489 case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
17490 StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
17491 "exec_lo" :
"exec_hi";
17494 CI->setConvergent();
17497 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray:
17498 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_h:
17499 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_l:
17500 case AMDGPU::BI__builtin_amdgcn_image_bvh_intersect_ray_lh: {
17510 RayOrigin = Builder.CreateShuffleVector(RayOrigin, RayOrigin,
17513 Builder.CreateShuffleVector(RayDir, RayDir,
ArrayRef<int>{0, 1, 2});
17514 RayInverseDir = Builder.CreateShuffleVector(RayInverseDir, RayInverseDir,
17517 Function *F =
CGM.
getIntrinsic(Intrinsic::amdgcn_image_bvh_intersect_ray,
17518 {NodePtr->getType(), RayDir->getType()});
17519 return Builder.CreateCall(F, {NodePtr, RayExtent, RayOrigin, RayDir,
17520 RayInverseDir, TextureDescr});
17523 case AMDGPU::BI__builtin_amdgcn_ds_bvh_stack_rtn: {
17525 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
17529 Value *
Call = Builder.CreateCall(F, Args);
17530 Value *Rtn = Builder.CreateExtractValue(Call, 0);
17531 Value *A = Builder.CreateExtractValue(Call, 1);
17533 Value *I0 = Builder.CreateInsertElement(PoisonValue::get(RetTy), Rtn,
17535 return Builder.CreateInsertElement(I0, A, 1);
17538 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
17539 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
17540 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
17541 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
17542 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
17543 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
17544 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
17545 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
17546 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
17547 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
17548 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
17549 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64: {
17555 unsigned ArgForMatchingRetType;
17556 unsigned BuiltinWMMAOp;
17558 switch (BuiltinID) {
17559 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w32:
17560 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_f16_w64:
17561 ArgForMatchingRetType = 2;
17562 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_f16;
17564 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w32:
17565 case AMDGPU::BI__builtin_amdgcn_wmma_f32_16x16x16_bf16_w64:
17566 ArgForMatchingRetType = 2;
17567 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f32_16x16x16_bf16;
17569 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w32:
17570 case AMDGPU::BI__builtin_amdgcn_wmma_f16_16x16x16_f16_w64:
17571 ArgForMatchingRetType = 2;
17572 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_f16_16x16x16_f16;
17574 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w32:
17575 case AMDGPU::BI__builtin_amdgcn_wmma_bf16_16x16x16_bf16_w64:
17576 ArgForMatchingRetType = 2;
17577 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_bf16_16x16x16_bf16;
17579 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w32:
17580 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu8_w64:
17581 ArgForMatchingRetType = 4;
17582 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu8;
17584 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w32:
17585 case AMDGPU::BI__builtin_amdgcn_wmma_i32_16x16x16_iu4_w64:
17586 ArgForMatchingRetType = 4;
17587 BuiltinWMMAOp = Intrinsic::amdgcn_wmma_i32_16x16x16_iu4;
17592 for (
int i = 0, e = E->
getNumArgs(); i != e; ++i)
17596 {Args[ArgForMatchingRetType]->getType()});
17598 return Builder.CreateCall(F, Args);
17602 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
17604 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
17606 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
17610 case AMDGPU::BI__builtin_amdgcn_workgroup_size_x:
17611 return EmitAMDGPUWorkGroupSize(*
this, 0);
17612 case AMDGPU::BI__builtin_amdgcn_workgroup_size_y:
17613 return EmitAMDGPUWorkGroupSize(*
this, 1);
17614 case AMDGPU::BI__builtin_amdgcn_workgroup_size_z:
17615 return EmitAMDGPUWorkGroupSize(*
this, 2);
17618 case AMDGPU::BI__builtin_amdgcn_grid_size_x:
17619 return EmitAMDGPUGridSize(*
this, 0);
17620 case AMDGPU::BI__builtin_amdgcn_grid_size_y:
17621 return EmitAMDGPUGridSize(*
this, 1);
17622 case AMDGPU::BI__builtin_amdgcn_grid_size_z:
17623 return EmitAMDGPUGridSize(*
this, 2);
17626 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
17627 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
17629 case AMDGPU::BI__builtin_r600_read_tidig_x:
17631 case AMDGPU::BI__builtin_r600_read_tidig_y:
17633 case AMDGPU::BI__builtin_r600_read_tidig_z:
17635 case AMDGPU::BI__builtin_amdgcn_alignbit: {
17640 return Builder.CreateCall(F, { Src0, Src1, Src2 });
17642 case AMDGPU::BI__builtin_amdgcn_fence: {
17645 return Builder.CreateFence(AO, SSID);
17647 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
17648 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
17649 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
17650 case AMDGPU::BI__builtin_amdgcn_atomic_dec64: {
17651 llvm::AtomicRMWInst::BinOp BinOp;
17652 switch (BuiltinID) {
17653 case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
17654 case AMDGPU::BI__builtin_amdgcn_atomic_inc64:
17655 BinOp = llvm::AtomicRMWInst::UIncWrap;
17657 case AMDGPU::BI__builtin_amdgcn_atomic_dec32:
17658 case AMDGPU::BI__builtin_amdgcn_atomic_dec64:
17659 BinOp = llvm::AtomicRMWInst::UDecWrap;
17671 PtrTy->castAs<
PointerType>()->getPointeeType().isVolatileQualified();
17673 llvm::AtomicRMWInst *RMW =
17674 Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID);
17676 RMW->setVolatile(
true);
17679 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
17680 case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtnl: {
17686 return Builder.CreateCall(F, {Arg});
17697 unsigned IntrinsicID,
17701 for (
unsigned I = 0; I < NumArgs; ++I)
17708 return CGF.
Builder.CreateExtractValue(Call, 0);
17713 switch (BuiltinID) {
17714 case SystemZ::BI__builtin_tbegin: {
17716 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
17718 return Builder.CreateCall(F, {TDB, Control});
17720 case SystemZ::BI__builtin_tbegin_nofloat: {
17722 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff0c);
17724 return Builder.CreateCall(F, {TDB, Control});
17726 case SystemZ::BI__builtin_tbeginc: {
17728 Value *Control = llvm::ConstantInt::get(
Int32Ty, 0xff08);
17730 return Builder.CreateCall(F, {TDB, Control});
17732 case SystemZ::BI__builtin_tabort: {
17735 return Builder.CreateCall(F, Builder.CreateSExt(
Data,
Int64Ty,
"tabort"));
17737 case SystemZ::BI__builtin_non_tx_store: {
17749 case SystemZ::BI__builtin_s390_vpopctb:
17750 case SystemZ::BI__builtin_s390_vpopcth:
17751 case SystemZ::BI__builtin_s390_vpopctf:
17752 case SystemZ::BI__builtin_s390_vpopctg: {
17756 return Builder.CreateCall(F,
X);
17759 case SystemZ::BI__builtin_s390_vclzb:
17760 case SystemZ::BI__builtin_s390_vclzh:
17761 case SystemZ::BI__builtin_s390_vclzf:
17762 case SystemZ::BI__builtin_s390_vclzg: {
17765 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17767 return Builder.CreateCall(F, {
X, Undef});
17770 case SystemZ::BI__builtin_s390_vctzb:
17771 case SystemZ::BI__builtin_s390_vctzh:
17772 case SystemZ::BI__builtin_s390_vctzf:
17773 case SystemZ::BI__builtin_s390_vctzg: {
17776 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
17778 return Builder.CreateCall(F, {
X, Undef});
17781 case SystemZ::BI__builtin_s390_vfsqsb:
17782 case SystemZ::BI__builtin_s390_vfsqdb: {
17785 if (Builder.getIsFPConstrained()) {
17786 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_sqrt, ResultType);
17787 return Builder.CreateConstrainedFPCall(F, {
X });
17790 return Builder.CreateCall(F,
X);
17793 case SystemZ::BI__builtin_s390_vfmasb:
17794 case SystemZ::BI__builtin_s390_vfmadb: {
17799 if (Builder.getIsFPConstrained()) {
17800 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17801 return Builder.CreateConstrainedFPCall(F, {
X, Y, Z});
17804 return Builder.CreateCall(F, {
X, Y, Z});
17807 case SystemZ::BI__builtin_s390_vfmssb:
17808 case SystemZ::BI__builtin_s390_vfmsdb: {
17813 if (Builder.getIsFPConstrained()) {
17814 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17815 return Builder.CreateConstrainedFPCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17818 return Builder.CreateCall(F, {
X, Y, Builder.CreateFNeg(Z,
"neg")});
17821 case SystemZ::BI__builtin_s390_vfnmasb:
17822 case SystemZ::BI__builtin_s390_vfnmadb: {
17827 if (Builder.getIsFPConstrained()) {
17828 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17829 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, Z}),
"neg");
17832 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, Z}),
"neg");
17835 case SystemZ::BI__builtin_s390_vfnmssb:
17836 case SystemZ::BI__builtin_s390_vfnmsdb: {
17841 if (Builder.getIsFPConstrained()) {
17842 Function *F =
CGM.
getIntrinsic(Intrinsic::experimental_constrained_fma, ResultType);
17843 Value *NegZ = Builder.CreateFNeg(Z,
"sub");
17844 return Builder.CreateFNeg(Builder.CreateConstrainedFPCall(F, {X, Y, NegZ}));
17847 Value *NegZ = Builder.CreateFNeg(Z,
"neg");
17848 return Builder.CreateFNeg(Builder.CreateCall(F, {X, Y, NegZ}));
17851 case SystemZ::BI__builtin_s390_vflpsb:
17852 case SystemZ::BI__builtin_s390_vflpdb: {
17856 return Builder.CreateCall(F,
X);
17858 case SystemZ::BI__builtin_s390_vflnsb:
17859 case SystemZ::BI__builtin_s390_vflndb: {
17863 return Builder.CreateFNeg(Builder.CreateCall(F,
X),
"neg");
17865 case SystemZ::BI__builtin_s390_vfisb:
17866 case SystemZ::BI__builtin_s390_vfidb: {
17874 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17876 switch (M4.getZExtValue()) {
17879 switch (M5.getZExtValue()) {
17881 case 0: ID = Intrinsic::rint;
17882 CI = Intrinsic::experimental_constrained_rint;
break;
17886 switch (M5.getZExtValue()) {
17888 case 0: ID = Intrinsic::nearbyint;
17889 CI = Intrinsic::experimental_constrained_nearbyint;
break;
17890 case 1: ID = Intrinsic::round;
17891 CI = Intrinsic::experimental_constrained_round;
break;
17892 case 5: ID = Intrinsic::trunc;
17893 CI = Intrinsic::experimental_constrained_trunc;
break;
17894 case 6: ID = Intrinsic::ceil;
17895 CI = Intrinsic::experimental_constrained_ceil;
break;
17896 case 7: ID = Intrinsic::floor;
17897 CI = Intrinsic::experimental_constrained_floor;
break;
17901 if (ID != Intrinsic::not_intrinsic) {
17902 if (Builder.getIsFPConstrained()) {
17904 return Builder.CreateConstrainedFPCall(F,
X);
17907 return Builder.CreateCall(F,
X);
17910 switch (BuiltinID) {
17911 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb;
break;
17912 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb;
break;
17913 default: llvm_unreachable(
"Unknown BuiltinID");
17918 return Builder.CreateCall(F, {
X, M4Value, M5Value});
17920 case SystemZ::BI__builtin_s390_vfmaxsb:
17921 case SystemZ::BI__builtin_s390_vfmaxdb: {
17929 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17931 switch (M4.getZExtValue()) {
17933 case 4: ID = Intrinsic::maxnum;
17934 CI = Intrinsic::experimental_constrained_maxnum;
break;
17936 if (ID != Intrinsic::not_intrinsic) {
17937 if (Builder.getIsFPConstrained()) {
17939 return Builder.CreateConstrainedFPCall(F, {
X, Y});
17942 return Builder.CreateCall(F, {
X, Y});
17945 switch (BuiltinID) {
17946 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb;
break;
17947 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb;
break;
17948 default: llvm_unreachable(
"Unknown BuiltinID");
17952 return Builder.CreateCall(F, {
X, Y, M4Value});
17954 case SystemZ::BI__builtin_s390_vfminsb:
17955 case SystemZ::BI__builtin_s390_vfmindb: {
17963 Intrinsic::ID ID = Intrinsic::not_intrinsic;
17965 switch (M4.getZExtValue()) {
17967 case 4: ID = Intrinsic::minnum;
17968 CI = Intrinsic::experimental_constrained_minnum;
break;
17970 if (ID != Intrinsic::not_intrinsic) {
17971 if (Builder.getIsFPConstrained()) {
17973 return Builder.CreateConstrainedFPCall(F, {
X, Y});
17976 return Builder.CreateCall(F, {
X, Y});
17979 switch (BuiltinID) {
17980 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb;
break;
17981 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb;
break;
17982 default: llvm_unreachable(
"Unknown BuiltinID");
17986 return Builder.CreateCall(F, {
X, Y, M4Value});
17989 case SystemZ::BI__builtin_s390_vlbrh:
17990 case SystemZ::BI__builtin_s390_vlbrf:
17991 case SystemZ::BI__builtin_s390_vlbrg: {
17995 return Builder.CreateCall(F,
X);
18000#define INTRINSIC_WITH_CC(NAME) \
18001 case SystemZ::BI__builtin_##NAME: \
18002 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E)
18081#undef INTRINSIC_WITH_CC
18090struct NVPTXMmaLdstInfo {
18091 unsigned NumResults;
18097#define MMA_INTR(geom_op_type, layout) \
18098 Intrinsic::nvvm_wmma_##geom_op_type##_##layout##_stride
18099#define MMA_LDST(n, geom_op_type) \
18100 { n, MMA_INTR(geom_op_type, col), MMA_INTR(geom_op_type, row) }
18102static NVPTXMmaLdstInfo getNVPTXMmaLdstInfo(
unsigned BuiltinID) {
18103 switch (BuiltinID) {
18105 case NVPTX::BI__hmma_m16n16k16_ld_a:
18106 return MMA_LDST(8, m16n16k16_load_a_f16);
18107 case NVPTX::BI__hmma_m16n16k16_ld_b:
18108 return MMA_LDST(8, m16n16k16_load_b_f16);
18109 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
18110 return MMA_LDST(4, m16n16k16_load_c_f16);
18111 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
18112 return MMA_LDST(8, m16n16k16_load_c_f32);
18113 case NVPTX::BI__hmma_m32n8k16_ld_a:
18114 return MMA_LDST(8, m32n8k16_load_a_f16);
18115 case NVPTX::BI__hmma_m32n8k16_ld_b:
18116 return MMA_LDST(8, m32n8k16_load_b_f16);
18117 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
18118 return MMA_LDST(4, m32n8k16_load_c_f16);
18119 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
18120 return MMA_LDST(8, m32n8k16_load_c_f32);
18121 case NVPTX::BI__hmma_m8n32k16_ld_a:
18122 return MMA_LDST(8, m8n32k16_load_a_f16);
18123 case NVPTX::BI__hmma_m8n32k16_ld_b:
18124 return MMA_LDST(8, m8n32k16_load_b_f16);
18125 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
18126 return MMA_LDST(4, m8n32k16_load_c_f16);
18127 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
18128 return MMA_LDST(8, m8n32k16_load_c_f32);
18131 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
18132 return MMA_LDST(2, m16n16k16_load_a_s8);
18133 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
18134 return MMA_LDST(2, m16n16k16_load_a_u8);
18135 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
18136 return MMA_LDST(2, m16n16k16_load_b_s8);
18137 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
18138 return MMA_LDST(2, m16n16k16_load_b_u8);
18139 case NVPTX::BI__imma_m16n16k16_ld_c:
18140 return MMA_LDST(8, m16n16k16_load_c_s32);
18141 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
18142 return MMA_LDST(4, m32n8k16_load_a_s8);
18143 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
18144 return MMA_LDST(4, m32n8k16_load_a_u8);
18145 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
18146 return MMA_LDST(1, m32n8k16_load_b_s8);
18147 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
18148 return MMA_LDST(1, m32n8k16_load_b_u8);
18149 case NVPTX::BI__imma_m32n8k16_ld_c:
18150 return MMA_LDST(8, m32n8k16_load_c_s32);
18151 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
18152 return MMA_LDST(1, m8n32k16_load_a_s8);
18153 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
18154 return MMA_LDST(1, m8n32k16_load_a_u8);
18155 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
18156 return MMA_LDST(4, m8n32k16_load_b_s8);
18157 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
18158 return MMA_LDST(4, m8n32k16_load_b_u8);
18159 case NVPTX::BI__imma_m8n32k16_ld_c:
18160 return MMA_LDST(8, m8n32k16_load_c_s32);
18164 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
18165 return {1, 0,
MMA_INTR(m8n8k32_load_a_s4, row)};
18166 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
18167 return {1, 0,
MMA_INTR(m8n8k32_load_a_u4, row)};
18168 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
18169 return {1,
MMA_INTR(m8n8k32_load_b_s4, col), 0};
18170 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
18171 return {1,
MMA_INTR(m8n8k32_load_b_u4, col), 0};
18172 case NVPTX::BI__imma_m8n8k32_ld_c:
18173 return MMA_LDST(2, m8n8k32_load_c_s32);
18174 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
18175 return {1, 0,
MMA_INTR(m8n8k128_load_a_b1, row)};
18176 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
18177 return {1,
MMA_INTR(m8n8k128_load_b_b1, col), 0};
18178 case NVPTX::BI__bmma_m8n8k128_ld_c:
18179 return MMA_LDST(2, m8n8k128_load_c_s32);
18182 case NVPTX::BI__dmma_m8n8k4_ld_a:
18183 return MMA_LDST(1, m8n8k4_load_a_f64);
18184 case NVPTX::BI__dmma_m8n8k4_ld_b:
18185 return MMA_LDST(1, m8n8k4_load_b_f64);
18186 case NVPTX::BI__dmma_m8n8k4_ld_c:
18187 return MMA_LDST(2, m8n8k4_load_c_f64);
18190 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
18191 return MMA_LDST(4, m16n16k16_load_a_bf16);
18192 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
18193 return MMA_LDST(4, m16n16k16_load_b_bf16);
18194 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
18195 return MMA_LDST(2, m8n32k16_load_a_bf16);
18196 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
18197 return MMA_LDST(8, m8n32k16_load_b_bf16);
18198 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
18199 return MMA_LDST(8, m32n8k16_load_a_bf16);
18200 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
18201 return MMA_LDST(2, m32n8k16_load_b_bf16);
18202 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
18203 return MMA_LDST(4, m16n16k8_load_a_tf32);
18204 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
18205 return MMA_LDST(4, m16n16k8_load_b_tf32);
18206 case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
18207 return MMA_LDST(8, m16n16k8_load_c_f32);
18213 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
18214 return MMA_LDST(4, m16n16k16_store_d_f16);
18215 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
18216 return MMA_LDST(8, m16n16k16_store_d_f32);
18217 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
18218 return MMA_LDST(4, m32n8k16_store_d_f16);
18219 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
18220 return MMA_LDST(8, m32n8k16_store_d_f32);
18221 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
18222 return MMA_LDST(4, m8n32k16_store_d_f16);
18223 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
18224 return MMA_LDST(8, m8n32k16_store_d_f32);
18229 case NVPTX::BI__imma_m16n16k16_st_c_i32:
18230 return MMA_LDST(8, m16n16k16_store_d_s32);
18231 case NVPTX::BI__imma_m32n8k16_st_c_i32:
18232 return MMA_LDST(8, m32n8k16_store_d_s32);
18233 case NVPTX::BI__imma_m8n32k16_st_c_i32:
18234 return MMA_LDST(8, m8n32k16_store_d_s32);
18235 case NVPTX::BI__imma_m8n8k32_st_c_i32:
18236 return MMA_LDST(2, m8n8k32_store_d_s32);
18237 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
18238 return MMA_LDST(2, m8n8k128_store_d_s32);
18241 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
18242 return MMA_LDST(2, m8n8k4_store_d_f64);
18245 case NVPTX::BI__mma_m16n16k8_st_c_f32:
18246 return MMA_LDST(8, m16n16k8_store_d_f32);
18249 llvm_unreachable(
"Unknown MMA builtin");
18256struct NVPTXMmaInfo {
18265 std::array<unsigned, 8> Variants;
18267 unsigned getMMAIntrinsic(
int Layout,
bool Satf) {
18268 unsigned Index = Layout + 4 * Satf;
18269 if (Index >= Variants.size())
18271 return Variants[Index];
18277static NVPTXMmaInfo getNVPTXMmaInfo(
unsigned BuiltinID) {
18279#define MMA_VARIANTS(geom, type) \
18280 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \
18281 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
18282 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \
18283 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type
18284#define MMA_SATF_VARIANTS(geom, type) \
18285 MMA_VARIANTS(geom, type), \
18286 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \
18287 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
18288 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \
18289 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite
18291#define MMA_VARIANTS_I4(geom, type) \
18293 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \
18297 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \
18301#define MMA_VARIANTS_B1_XOR(geom, type) \
18303 Intrinsic::nvvm_wmma_##geom##_mma_xor_popc_row_col_##type, \
18310#define MMA_VARIANTS_B1_AND(geom, type) \
18312 Intrinsic::nvvm_wmma_##geom##_mma_and_popc_row_col_##type, \
18320 switch (BuiltinID) {
18324 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
18326 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
18328 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
18330 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
18332 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
18334 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
18336 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
18338 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
18340 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
18342 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
18344 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
18346 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
18350 case NVPTX::BI__imma_m16n16k16_mma_s8:
18352 case NVPTX::BI__imma_m16n16k16_mma_u8:
18354 case NVPTX::BI__imma_m32n8k16_mma_s8:
18356 case NVPTX::BI__imma_m32n8k16_mma_u8:
18358 case NVPTX::BI__imma_m8n32k16_mma_s8:
18360 case NVPTX::BI__imma_m8n32k16_mma_u8:
18364 case NVPTX::BI__imma_m8n8k32_mma_s4:
18366 case NVPTX::BI__imma_m8n8k32_mma_u4:
18368 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
18370 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
18374 case NVPTX::BI__dmma_m8n8k4_mma_f64:
18378 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
18379 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k16, bf16)}}};
18380 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
18381 return {2, 8, 8, 8, {{
MMA_VARIANTS(m8n32k16, bf16)}}};
18382 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
18383 return {8, 2, 8, 8, {{
MMA_VARIANTS(m32n8k16, bf16)}}};
18384 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
18385 return {4, 4, 8, 8, {{
MMA_VARIANTS(m16n16k8, tf32)}}};
18387 llvm_unreachable(
"Unexpected builtin ID.");
18390#undef MMA_SATF_VARIANTS
18391#undef MMA_VARIANTS_I4
18392#undef MMA_VARIANTS_B1_AND
18393#undef MMA_VARIANTS_B1_XOR
18402 return CGF.
Builder.CreateCall(
18404 {Ptr, ConstantInt::get(CGF.Builder.getInt32Ty(), Align.getQuantity())});
18410 llvm::Type *ElemTy =
18412 return CGF.
Builder.CreateCall(
18414 {Ptr, CGF.EmitScalarExpr(E->getArg(1))});
18417static Value *MakeCpAsync(
unsigned IntrinsicID,
unsigned IntrinsicIDS,
18422 {CGF.EmitScalarExpr(E->getArg(0)),
18423 CGF.EmitScalarExpr(E->getArg(1)),
18424 CGF.EmitScalarExpr(E->getArg(2))})
18426 {CGF.EmitScalarExpr(E->getArg(0)),
18427 CGF.EmitScalarExpr(E->getArg(1))});
18430static Value *MakeHalfType(
unsigned IntrinsicID,
unsigned BuiltinID,
18433 if (!(
C.getLangOpts().NativeHalfType ||
18434 !
C.getTargetInfo().useFP16ConversionIntrinsics())) {
18436 " requires native half type support.");
18440 if (IntrinsicID == Intrinsic::nvvm_ldg_global_f ||
18441 IntrinsicID == Intrinsic::nvvm_ldu_global_f)
18442 return MakeLdgLdu(IntrinsicID, CGF, E);
18446 auto *FTy = F->getFunctionType();
18447 unsigned ICEArguments = 0;
18449 C.GetBuiltinType(BuiltinID,
Error, &ICEArguments);
18451 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
18452 assert((ICEArguments & (1 << i)) == 0);
18454 auto *PTy = FTy->getParamType(i);
18455 if (PTy != ArgValue->
getType())
18456 ArgValue = CGF.
Builder.CreateBitCast(ArgValue, PTy);
18457 Args.push_back(ArgValue);
18460 return CGF.
Builder.CreateCall(F, Args);
18466 switch (BuiltinID) {
18467 case NVPTX::BI__nvvm_atom_add_gen_i:
18468 case NVPTX::BI__nvvm_atom_add_gen_l:
18469 case NVPTX::BI__nvvm_atom_add_gen_ll:
18472 case NVPTX::BI__nvvm_atom_sub_gen_i:
18473 case NVPTX::BI__nvvm_atom_sub_gen_l:
18474 case NVPTX::BI__nvvm_atom_sub_gen_ll:
18477 case NVPTX::BI__nvvm_atom_and_gen_i:
18478 case NVPTX::BI__nvvm_atom_and_gen_l:
18479 case NVPTX::BI__nvvm_atom_and_gen_ll:
18482 case NVPTX::BI__nvvm_atom_or_gen_i:
18483 case NVPTX::BI__nvvm_atom_or_gen_l:
18484 case NVPTX::BI__nvvm_atom_or_gen_ll:
18487 case NVPTX::BI__nvvm_atom_xor_gen_i:
18488 case NVPTX::BI__nvvm_atom_xor_gen_l:
18489 case NVPTX::BI__nvvm_atom_xor_gen_ll:
18492 case NVPTX::BI__nvvm_atom_xchg_gen_i:
18493 case NVPTX::BI__nvvm_atom_xchg_gen_l:
18494 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
18497 case NVPTX::BI__nvvm_atom_max_gen_i:
18498 case NVPTX::BI__nvvm_atom_max_gen_l:
18499 case NVPTX::BI__nvvm_atom_max_gen_ll:
18502 case NVPTX::BI__nvvm_atom_max_gen_ui:
18503 case NVPTX::BI__nvvm_atom_max_gen_ul:
18504 case NVPTX::BI__nvvm_atom_max_gen_ull:
18507 case NVPTX::BI__nvvm_atom_min_gen_i:
18508 case NVPTX::BI__nvvm_atom_min_gen_l:
18509 case NVPTX::BI__nvvm_atom_min_gen_ll:
18512 case NVPTX::BI__nvvm_atom_min_gen_ui:
18513 case NVPTX::BI__nvvm_atom_min_gen_ul:
18514 case NVPTX::BI__nvvm_atom_min_gen_ull:
18517 case NVPTX::BI__nvvm_atom_cas_gen_i:
18518 case NVPTX::BI__nvvm_atom_cas_gen_l:
18519 case NVPTX::BI__nvvm_atom_cas_gen_ll:
18524 case NVPTX::BI__nvvm_atom_add_gen_f:
18525 case NVPTX::BI__nvvm_atom_add_gen_d: {
18528 return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, Ptr, Val,
18529 AtomicOrdering::SequentiallyConsistent);
18532 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
18535 Function *FnALI32 =
18537 return Builder.CreateCall(FnALI32, {Ptr, Val});
18540 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
18543 Function *FnALD32 =
18545 return Builder.CreateCall(FnALD32, {Ptr, Val});
18548 case NVPTX::BI__nvvm_ldg_c:
18549 case NVPTX::BI__nvvm_ldg_sc:
18550 case NVPTX::BI__nvvm_ldg_c2:
18551 case NVPTX::BI__nvvm_ldg_sc2:
18552 case NVPTX::BI__nvvm_ldg_c4:
18553 case NVPTX::BI__nvvm_ldg_sc4:
18554 case NVPTX::BI__nvvm_ldg_s:
18555 case NVPTX::BI__nvvm_ldg_s2:
18556 case NVPTX::BI__nvvm_ldg_s4:
18557 case NVPTX::BI__nvvm_ldg_i:
18558 case NVPTX::BI__nvvm_ldg_i2:
18559 case NVPTX::BI__nvvm_ldg_i4:
18560 case NVPTX::BI__nvvm_ldg_l:
18561 case NVPTX::BI__nvvm_ldg_l2:
18562 case NVPTX::BI__nvvm_ldg_ll:
18563 case NVPTX::BI__nvvm_ldg_ll2:
18564 case NVPTX::BI__nvvm_ldg_uc:
18565 case NVPTX::BI__nvvm_ldg_uc2:
18566 case NVPTX::BI__nvvm_ldg_uc4:
18567 case NVPTX::BI__nvvm_ldg_us:
18568 case NVPTX::BI__nvvm_ldg_us2:
18569 case NVPTX::BI__nvvm_ldg_us4:
18570 case NVPTX::BI__nvvm_ldg_ui:
18571 case NVPTX::BI__nvvm_ldg_ui2:
18572 case NVPTX::BI__nvvm_ldg_ui4:
18573 case NVPTX::BI__nvvm_ldg_ul:
18574 case NVPTX::BI__nvvm_ldg_ul2:
18575 case NVPTX::BI__nvvm_ldg_ull:
18576 case NVPTX::BI__nvvm_ldg_ull2:
18580 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_i, *
this, E);
18581 case NVPTX::BI__nvvm_ldg_f:
18582 case NVPTX::BI__nvvm_ldg_f2:
18583 case NVPTX::BI__nvvm_ldg_f4:
18584 case NVPTX::BI__nvvm_ldg_d:
18585 case NVPTX::BI__nvvm_ldg_d2:
18586 return MakeLdgLdu(Intrinsic::nvvm_ldg_global_f, *
this, E);
18588 case NVPTX::BI__nvvm_ldu_c:
18589 case NVPTX::BI__nvvm_ldu_sc:
18590 case NVPTX::BI__nvvm_ldu_c2:
18591 case NVPTX::BI__nvvm_ldu_sc2:
18592 case NVPTX::BI__nvvm_ldu_c4:
18593 case NVPTX::BI__nvvm_ldu_sc4:
18594 case NVPTX::BI__nvvm_ldu_s:
18595 case NVPTX::BI__nvvm_ldu_s2:
18596 case NVPTX::BI__nvvm_ldu_s4:
18597 case NVPTX::BI__nvvm_ldu_i:
18598 case NVPTX::BI__nvvm_ldu_i2:
18599 case NVPTX::BI__nvvm_ldu_i4:
18600 case NVPTX::BI__nvvm_ldu_l:
18601 case NVPTX::BI__nvvm_ldu_l2:
18602 case NVPTX::BI__nvvm_ldu_ll:
18603 case NVPTX::BI__nvvm_ldu_ll2:
18604 case NVPTX::BI__nvvm_ldu_uc:
18605 case NVPTX::BI__nvvm_ldu_uc2:
18606 case NVPTX::BI__nvvm_ldu_uc4:
18607 case NVPTX::BI__nvvm_ldu_us:
18608 case NVPTX::BI__nvvm_ldu_us2:
18609 case NVPTX::BI__nvvm_ldu_us4:
18610 case NVPTX::BI__nvvm_ldu_ui:
18611 case NVPTX::BI__nvvm_ldu_ui2:
18612 case NVPTX::BI__nvvm_ldu_ui4:
18613 case NVPTX::BI__nvvm_ldu_ul:
18614 case NVPTX::BI__nvvm_ldu_ul2:
18615 case NVPTX::BI__nvvm_ldu_ull:
18616 case NVPTX::BI__nvvm_ldu_ull2:
18617 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_i, *
this, E);
18618 case NVPTX::BI__nvvm_ldu_f:
18619 case NVPTX::BI__nvvm_ldu_f2:
18620 case NVPTX::BI__nvvm_ldu_f4:
18621 case NVPTX::BI__nvvm_ldu_d:
18622 case NVPTX::BI__nvvm_ldu_d2:
18623 return MakeLdgLdu(Intrinsic::nvvm_ldu_global_f, *
this, E);
18625 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
18626 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
18627 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
18628 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta, *
this, E);
18629 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
18630 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
18631 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
18632 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys, *
this, E);
18633 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
18634 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
18635 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta, *
this, E);
18636 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
18637 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
18638 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys, *
this, E);
18639 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
18640 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
18641 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
18642 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta, *
this, E);
18643 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
18644 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
18645 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
18646 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys, *
this, E);
18647 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
18648 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
18649 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
18650 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
18651 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
18652 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
18653 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta, *
this, E);
18654 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
18655 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
18656 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
18657 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
18658 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
18659 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
18660 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys, *
this, E);
18661 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
18662 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
18663 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
18664 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
18665 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
18666 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
18667 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta, *
this, E);
18668 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
18669 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
18670 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
18671 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
18672 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
18673 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
18674 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys, *
this, E);
18675 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
18676 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta, *
this, E);
18677 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
18678 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta, *
this, E);
18679 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
18680 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys, *
this, E);
18681 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
18682 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys, *
this, E);
18683 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
18684 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
18685 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
18686 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta, *
this, E);
18687 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
18688 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
18689 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
18690 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys, *
this, E);
18691 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
18692 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
18693 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
18694 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta, *
this, E);
18695 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
18696 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
18697 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
18698 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys, *
this, E);
18699 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
18700 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
18701 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
18702 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta, *
this, E);
18703 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
18704 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
18705 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
18706 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys, *
this, E);
18707 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
18708 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
18709 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
18711 llvm::Type *ElemTy =
18713 return Builder.CreateCall(
18715 Intrinsic::nvvm_atomic_cas_gen_i_cta, {ElemTy, Ptr->getType()}),
18716 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
18718 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
18719 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
18720 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
18722 llvm::Type *ElemTy =
18724 return Builder.CreateCall(
18726 Intrinsic::nvvm_atomic_cas_gen_i_sys, {ElemTy, Ptr->getType()}),
18727 {Ptr, EmitScalarExpr(E->getArg(1)), EmitScalarExpr(E->getArg(2))});
18729 case NVPTX::BI__nvvm_match_all_sync_i32p:
18730 case NVPTX::BI__nvvm_match_all_sync_i64p: {
18734 Value *ResultPair = Builder.CreateCall(
18736 ? Intrinsic::nvvm_match_all_sync_i32p
18737 : Intrinsic::nvvm_match_all_sync_i64p),
18739 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
18741 Builder.CreateStore(Pred, PredOutPtr);
18742 return Builder.CreateExtractValue(ResultPair, 0);
18746 case NVPTX::BI__hmma_m16n16k16_ld_a:
18747 case NVPTX::BI__hmma_m16n16k16_ld_b:
18748 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
18749 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
18750 case NVPTX::BI__hmma_m32n8k16_ld_a:
18751 case NVPTX::BI__hmma_m32n8k16_ld_b:
18752 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
18753 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
18754 case NVPTX::BI__hmma_m8n32k16_ld_a:
18755 case NVPTX::BI__hmma_m8n32k16_ld_b:
18756 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
18757 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
18759 case NVPTX::BI__imma_m16n16k16_ld_a_s8:
18760 case NVPTX::BI__imma_m16n16k16_ld_a_u8:
18761 case NVPTX::BI__imma_m16n16k16_ld_b_s8:
18762 case NVPTX::BI__imma_m16n16k16_ld_b_u8:
18763 case NVPTX::BI__imma_m16n16k16_ld_c:
18764 case NVPTX::BI__imma_m32n8k16_ld_a_s8:
18765 case NVPTX::BI__imma_m32n8k16_ld_a_u8:
18766 case NVPTX::BI__imma_m32n8k16_ld_b_s8:
18767 case NVPTX::BI__imma_m32n8k16_ld_b_u8:
18768 case NVPTX::BI__imma_m32n8k16_ld_c:
18769 case NVPTX::BI__imma_m8n32k16_ld_a_s8:
18770 case NVPTX::BI__imma_m8n32k16_ld_a_u8:
18771 case NVPTX::BI__imma_m8n32k16_ld_b_s8:
18772 case NVPTX::BI__imma_m8n32k16_ld_b_u8:
18773 case NVPTX::BI__imma_m8n32k16_ld_c:
18775 case NVPTX::BI__imma_m8n8k32_ld_a_s4:
18776 case NVPTX::BI__imma_m8n8k32_ld_a_u4:
18777 case NVPTX::BI__imma_m8n8k32_ld_b_s4:
18778 case NVPTX::BI__imma_m8n8k32_ld_b_u4:
18779 case NVPTX::BI__imma_m8n8k32_ld_c:
18780 case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
18781 case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
18782 case NVPTX::BI__bmma_m8n8k128_ld_c:
18784 case NVPTX::BI__dmma_m8n8k4_ld_a:
18785 case NVPTX::BI__dmma_m8n8k4_ld_b:
18786 case NVPTX::BI__dmma_m8n8k4_ld_c:
18788 case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
18789 case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
18790 case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
18791 case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
18792 case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
18793 case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
18794 case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
18795 case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
18796 case NVPTX::BI__mma_tf32_m16n16k8_ld_c: {
18800 std::optional<llvm::APSInt> isColMajorArg =
18802 if (!isColMajorArg)
18804 bool isColMajor = isColMajorArg->getSExtValue();
18805 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
18806 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
18814 assert(II.NumResults);
18815 if (II.NumResults == 1) {
18819 for (
unsigned i = 0; i < II.NumResults; ++i) {
18820 Builder.CreateAlignedStore(
18821 Builder.CreateBitCast(Builder.CreateExtractValue(
Result, i),
18824 llvm::ConstantInt::get(
IntTy, i)),
18831 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
18832 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
18833 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
18834 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
18835 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
18836 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
18837 case NVPTX::BI__imma_m16n16k16_st_c_i32:
18838 case NVPTX::BI__imma_m32n8k16_st_c_i32:
18839 case NVPTX::BI__imma_m8n32k16_st_c_i32:
18840 case NVPTX::BI__imma_m8n8k32_st_c_i32:
18841 case NVPTX::BI__bmma_m8n8k128_st_c_i32:
18842 case NVPTX::BI__dmma_m8n8k4_st_c_f64:
18843 case NVPTX::BI__mma_m16n16k8_st_c_f32: {
18847 std::optional<llvm::APSInt> isColMajorArg =
18849 if (!isColMajorArg)
18851 bool isColMajor = isColMajorArg->getSExtValue();
18852 NVPTXMmaLdstInfo II = getNVPTXMmaLdstInfo(BuiltinID);
18853 unsigned IID = isColMajor ? II.IID_col : II.IID_row;
18856 Function *Intrinsic =
18858 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
18860 for (
unsigned i = 0; i < II.NumResults; ++i) {
18861 Value *
V = Builder.CreateAlignedLoad(
18864 llvm::ConstantInt::get(
IntTy, i)),
18866 Values.push_back(Builder.CreateBitCast(
V, ParamType));
18868 Values.push_back(Ldm);
18869 Value *
Result = Builder.CreateCall(Intrinsic, Values);
18875 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
18876 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
18877 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
18878 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
18879 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
18880 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
18881 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
18882 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
18883 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
18884 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
18885 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
18886 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
18887 case NVPTX::BI__imma_m16n16k16_mma_s8:
18888 case NVPTX::BI__imma_m16n16k16_mma_u8:
18889 case NVPTX::BI__imma_m32n8k16_mma_s8:
18890 case NVPTX::BI__imma_m32n8k16_mma_u8:
18891 case NVPTX::BI__imma_m8n32k16_mma_s8:
18892 case NVPTX::BI__imma_m8n32k16_mma_u8:
18893 case NVPTX::BI__imma_m8n8k32_mma_s4:
18894 case NVPTX::BI__imma_m8n8k32_mma_u4:
18895 case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
18896 case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
18897 case NVPTX::BI__dmma_m8n8k4_mma_f64:
18898 case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
18899 case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
18900 case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
18901 case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: {
18906 std::optional<llvm::APSInt> LayoutArg =
18910 int Layout = LayoutArg->getSExtValue();
18911 if (Layout < 0 || Layout > 3)
18913 llvm::APSInt SatfArg;
18914 if (BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1 ||
18915 BuiltinID == NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1)
18917 else if (std::optional<llvm::APSInt> OptSatfArg =
18919 SatfArg = *OptSatfArg;
18922 bool Satf = SatfArg.getSExtValue();
18923 NVPTXMmaInfo MI = getNVPTXMmaInfo(BuiltinID);
18924 unsigned IID = MI.getMMAIntrinsic(Layout, Satf);
18930 llvm::Type *AType = Intrinsic->getFunctionType()->getParamType(0);
18932 for (
unsigned i = 0; i < MI.NumEltsA; ++i) {
18933 Value *
V = Builder.CreateAlignedLoad(
18936 llvm::ConstantInt::get(
IntTy, i)),
18938 Values.push_back(Builder.CreateBitCast(
V, AType));
18941 llvm::Type *BType = Intrinsic->getFunctionType()->getParamType(MI.NumEltsA);
18942 for (
unsigned i = 0; i < MI.NumEltsB; ++i) {
18943 Value *
V = Builder.CreateAlignedLoad(
18946 llvm::ConstantInt::get(
IntTy, i)),
18948 Values.push_back(Builder.CreateBitCast(
V, BType));
18951 llvm::Type *CType =
18952 Intrinsic->getFunctionType()->getParamType(MI.NumEltsA + MI.NumEltsB);
18953 for (
unsigned i = 0; i < MI.NumEltsC; ++i) {
18954 Value *
V = Builder.CreateAlignedLoad(
18957 llvm::ConstantInt::get(
IntTy, i)),
18959 Values.push_back(Builder.CreateBitCast(
V, CType));
18961 Value *
Result = Builder.CreateCall(Intrinsic, Values);
18963 for (
unsigned i = 0; i < MI.NumEltsD; ++i)
18964 Builder.CreateAlignedStore(
18965 Builder.CreateBitCast(Builder.CreateExtractValue(
Result, i), DType),
18967 llvm::ConstantInt::get(
IntTy, i)),
18972 case NVPTX::BI__nvvm_ex2_approx_f16:
18973 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16, BuiltinID, E, *
this);
18974 case NVPTX::BI__nvvm_ex2_approx_f16x2:
18975 return MakeHalfType(Intrinsic::nvvm_ex2_approx_f16x2, BuiltinID, E, *
this);
18976 case NVPTX::BI__nvvm_ff2f16x2_rn:
18977 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn, BuiltinID, E, *
this);
18978 case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
18979 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rn_relu, BuiltinID, E, *
this);
18980 case NVPTX::BI__nvvm_ff2f16x2_rz:
18981 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz, BuiltinID, E, *
this);
18982 case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
18983 return MakeHalfType(Intrinsic::nvvm_ff2f16x2_rz_relu, BuiltinID, E, *
this);
18984 case NVPTX::BI__nvvm_fma_rn_f16:
18985 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16, BuiltinID, E, *
this);
18986 case NVPTX::BI__nvvm_fma_rn_f16x2:
18987 return MakeHalfType(Intrinsic::nvvm_fma_rn_f16x2, BuiltinID, E, *
this);
18988 case NVPTX::BI__nvvm_fma_rn_ftz_f16:
18989 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16, BuiltinID, E, *
this);
18990 case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
18991 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_f16x2, BuiltinID, E, *
this);
18992 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
18993 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16, BuiltinID, E,
18995 case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
18996 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_relu_f16x2, BuiltinID, E,
18998 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
18999 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16, BuiltinID, E,
19001 case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
19002 return MakeHalfType(Intrinsic::nvvm_fma_rn_ftz_sat_f16x2, BuiltinID, E,
19004 case NVPTX::BI__nvvm_fma_rn_relu_f16:
19005 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16, BuiltinID, E, *
this);
19006 case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
19007 return MakeHalfType(Intrinsic::nvvm_fma_rn_relu_f16x2, BuiltinID, E, *
this);
19008 case NVPTX::BI__nvvm_fma_rn_sat_f16:
19009 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16, BuiltinID, E, *
this);
19010 case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
19011 return MakeHalfType(Intrinsic::nvvm_fma_rn_sat_f16x2, BuiltinID, E, *
this);
19012 case NVPTX::BI__nvvm_fmax_f16:
19013 return MakeHalfType(Intrinsic::nvvm_fmax_f16, BuiltinID, E, *
this);
19014 case NVPTX::BI__nvvm_fmax_f16x2:
19015 return MakeHalfType(Intrinsic::nvvm_fmax_f16x2, BuiltinID, E, *
this);
19016 case NVPTX::BI__nvvm_fmax_ftz_f16:
19017 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16, BuiltinID, E, *
this);
19018 case NVPTX::BI__nvvm_fmax_ftz_f16x2:
19019 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_f16x2, BuiltinID, E, *
this);
19020 case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
19021 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16, BuiltinID, E, *
this);
19022 case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
19023 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_f16x2, BuiltinID, E,
19025 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
19026 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16, BuiltinID,
19028 case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
19029 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f16x2,
19030 BuiltinID, E, *
this);
19031 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
19032 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16, BuiltinID, E,
19034 case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
19035 return MakeHalfType(Intrinsic::nvvm_fmax_ftz_xorsign_abs_f16x2, BuiltinID,
19037 case NVPTX::BI__nvvm_fmax_nan_f16:
19038 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16, BuiltinID, E, *
this);
19039 case NVPTX::BI__nvvm_fmax_nan_f16x2:
19040 return MakeHalfType(Intrinsic::nvvm_fmax_nan_f16x2, BuiltinID, E, *
this);
19041 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
19042 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16, BuiltinID, E,
19044 case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
19045 return MakeHalfType(Intrinsic::nvvm_fmax_nan_xorsign_abs_f16x2, BuiltinID,
19047 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
19048 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16, BuiltinID, E,
19050 case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
19051 return MakeHalfType(Intrinsic::nvvm_fmax_xorsign_abs_f16x2, BuiltinID, E,
19053 case NVPTX::BI__nvvm_fmin_f16:
19054 return MakeHalfType(Intrinsic::nvvm_fmin_f16, BuiltinID, E, *
this);
19055 case NVPTX::BI__nvvm_fmin_f16x2:
19056 return MakeHalfType(Intrinsic::nvvm_fmin_f16x2, BuiltinID, E, *
this);
19057 case NVPTX::BI__nvvm_fmin_ftz_f16:
19058 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16, BuiltinID, E, *
this);
19059 case NVPTX::BI__nvvm_fmin_ftz_f16x2:
19060 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_f16x2, BuiltinID, E, *
this);
19061 case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
19062 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16, BuiltinID, E, *
this);
19063 case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
19064 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_f16x2, BuiltinID, E,
19066 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
19067 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16, BuiltinID,
19069 case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
19070 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f16x2,
19071 BuiltinID, E, *
this);
19072 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
19073 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16, BuiltinID, E,
19075 case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
19076 return MakeHalfType(Intrinsic::nvvm_fmin_ftz_xorsign_abs_f16x2, BuiltinID,
19078 case NVPTX::BI__nvvm_fmin_nan_f16:
19079 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16, BuiltinID, E, *
this);
19080 case NVPTX::BI__nvvm_fmin_nan_f16x2:
19081 return MakeHalfType(Intrinsic::nvvm_fmin_nan_f16x2, BuiltinID, E, *
this);
19082 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
19083 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16, BuiltinID, E,
19085 case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
19086 return MakeHalfType(Intrinsic::nvvm_fmin_nan_xorsign_abs_f16x2, BuiltinID,
19088 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
19089 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16, BuiltinID, E,
19091 case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
19092 return MakeHalfType(Intrinsic::nvvm_fmin_xorsign_abs_f16x2, BuiltinID, E,
19094 case NVPTX::BI__nvvm_ldg_h:
19095 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
19096 case NVPTX::BI__nvvm_ldg_h2:
19097 return MakeHalfType(Intrinsic::nvvm_ldg_global_f, BuiltinID, E, *
this);
19098 case NVPTX::BI__nvvm_ldu_h:
19099 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
19100 case NVPTX::BI__nvvm_ldu_h2: {
19101 return MakeHalfType(Intrinsic::nvvm_ldu_global_f, BuiltinID, E, *
this);
19103 case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
19104 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_4,
19105 Intrinsic::nvvm_cp_async_ca_shared_global_4_s, *
this, E,
19107 case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
19108 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_8,
19109 Intrinsic::nvvm_cp_async_ca_shared_global_8_s, *
this, E,
19111 case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
19112 return MakeCpAsync(Intrinsic::nvvm_cp_async_ca_shared_global_16,
19113 Intrinsic::nvvm_cp_async_ca_shared_global_16_s, *
this, E,
19115 case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
19116 return MakeCpAsync(Intrinsic::nvvm_cp_async_cg_shared_global_16,
19117 Intrinsic::nvvm_cp_async_cg_shared_global_16_s, *
this, E,
19119 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
19120 return Builder.CreateCall(
19122 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
19123 return Builder.CreateCall(
19125 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
19126 return Builder.CreateCall(
19128 case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
19129 return Builder.CreateCall(
19131 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
19132 return Builder.CreateCall(
19134 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
19135 return Builder.CreateCall(
19137 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
19138 return Builder.CreateCall(
19140 case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
19141 return Builder.CreateCall(
19143 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
19144 return Builder.CreateCall(
19146 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
19147 return Builder.CreateCall(
19149 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
19150 return Builder.CreateCall(
19152 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
19153 return Builder.CreateCall(
19155 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
19156 return Builder.CreateCall(
19158 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
19159 return Builder.CreateCall(
19161 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
19162 return Builder.CreateCall(
19164 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
19165 return Builder.CreateCall(
19167 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
19168 return Builder.CreateCall(
19170 case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
19171 return Builder.CreateCall(
19173 case NVPTX::BI__nvvm_is_explicit_cluster:
19174 return Builder.CreateCall(
19176 case NVPTX::BI__nvvm_isspacep_shared_cluster:
19177 return Builder.CreateCall(
19180 case NVPTX::BI__nvvm_mapa:
19181 return Builder.CreateCall(
19183 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
19184 case NVPTX::BI__nvvm_mapa_shared_cluster:
19185 return Builder.CreateCall(
19187 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
19188 case NVPTX::BI__nvvm_getctarank:
19189 return Builder.CreateCall(
19192 case NVPTX::BI__nvvm_getctarank_shared_cluster:
19193 return Builder.CreateCall(
19196 case NVPTX::BI__nvvm_barrier_cluster_arrive:
19197 return Builder.CreateCall(
19199 case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
19200 return Builder.CreateCall(
19202 case NVPTX::BI__nvvm_barrier_cluster_wait:
19203 return Builder.CreateCall(
19205 case NVPTX::BI__nvvm_fence_sc_cluster:
19206 return Builder.CreateCall(
19214struct BuiltinAlignArgs {
19215 llvm::Value *Src =
nullptr;
19216 llvm::Type *SrcType =
nullptr;
19217 llvm::Value *Alignment =
nullptr;
19218 llvm::Value *Mask =
nullptr;
19219 llvm::IntegerType *IntType =
nullptr;
19227 SrcType = Src->getType();
19228 if (SrcType->isPointerTy()) {
19229 IntType = IntegerType::get(
19233 assert(SrcType->isIntegerTy());
19237 Alignment = CGF.
Builder.CreateZExtOrTrunc(Alignment, IntType,
"alignment");
19238 auto *One = llvm::ConstantInt::get(IntType, 1);
19239 Mask = CGF.
Builder.CreateSub(Alignment, One,
"mask");
19246 BuiltinAlignArgs Args(E, *
this);
19247 llvm::Value *SrcAddress = Args.Src;
19248 if (Args.SrcType->isPointerTy())
19250 Builder.CreateBitOrPointerCast(Args.Src, Args.IntType,
"src_addr");
19252 Builder.CreateAnd(SrcAddress, Args.Mask,
"set_bits"),
19253 llvm::Constant::getNullValue(Args.IntType),
"is_aligned"));
19261 BuiltinAlignArgs Args(E, *
this);
19262 llvm::Value *SrcAddr = Args.Src;
19263 if (Args.Src->getType()->isPointerTy())
19264 SrcAddr = Builder.CreatePtrToInt(Args.Src, Args.IntType,
"intptr");
19265 llvm::Value *SrcForMask = SrcAddr;
19271 SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask,
"over_boundary");
19274 llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask,
"inverted_mask");
19276 Builder.CreateAnd(SrcForMask, InvertedMask,
"aligned_result");
19277 if (Args.Src->getType()->isPointerTy()) {
19282 Result->setName(
"aligned_intptr");
19283 llvm::Value *Difference = Builder.CreateSub(
Result, SrcAddr,
"diff");
19288 Builder.CreateGEP(
Int8Ty, Args.Src, Difference,
"aligned_result");
19298 assert(
Result->getType() == Args.SrcType);
19304 switch (BuiltinID) {
19305 case WebAssembly::BI__builtin_wasm_memory_size: {
19310 return Builder.CreateCall(Callee, I);
19312 case WebAssembly::BI__builtin_wasm_memory_grow: {
19318 return Builder.CreateCall(Callee, Args);
19320 case WebAssembly::BI__builtin_wasm_tls_size: {
19323 return Builder.CreateCall(Callee);
19325 case WebAssembly::BI__builtin_wasm_tls_align: {
19328 return Builder.CreateCall(Callee);
19330 case WebAssembly::BI__builtin_wasm_tls_base: {
19332 return Builder.CreateCall(Callee);
19334 case WebAssembly::BI__builtin_wasm_throw: {
19338 return Builder.CreateCall(Callee, {Tag, Obj});
19340 case WebAssembly::BI__builtin_wasm_rethrow: {
19342 return Builder.CreateCall(Callee);
19344 case WebAssembly::BI__builtin_wasm_memory_atomic_wait32: {
19349 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
19351 case WebAssembly::BI__builtin_wasm_memory_atomic_wait64: {
19356 return Builder.CreateCall(Callee, {Addr,
Expected, Timeout});
19358 case WebAssembly::BI__builtin_wasm_memory_atomic_notify: {
19362 return Builder.CreateCall(Callee, {Addr, Count});
19364 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f32:
19365 case WebAssembly::BI__builtin_wasm_trunc_s_i32_f64:
19366 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f32:
19367 case WebAssembly::BI__builtin_wasm_trunc_s_i64_f64: {
19372 return Builder.CreateCall(Callee, {Src});
19374 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f32:
19375 case WebAssembly::BI__builtin_wasm_trunc_u_i32_f64:
19376 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f32:
19377 case WebAssembly::BI__builtin_wasm_trunc_u_i64_f64: {
19382 return Builder.CreateCall(Callee, {Src});
19384 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
19385 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
19386 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
19387 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
19388 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4: {
19393 return Builder.CreateCall(Callee, {Src});
19395 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
19396 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
19397 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
19398 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
19399 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4: {
19404 return Builder.CreateCall(Callee, {Src});
19406 case WebAssembly::BI__builtin_wasm_min_f32:
19407 case WebAssembly::BI__builtin_wasm_min_f64:
19408 case WebAssembly::BI__builtin_wasm_min_f32x4:
19409 case WebAssembly::BI__builtin_wasm_min_f64x2: {
19414 return Builder.CreateCall(Callee, {LHS, RHS});
19416 case WebAssembly::BI__builtin_wasm_max_f32:
19417 case WebAssembly::BI__builtin_wasm_max_f64:
19418 case WebAssembly::BI__builtin_wasm_max_f32x4:
19419 case WebAssembly::BI__builtin_wasm_max_f64x2: {
19424 return Builder.CreateCall(Callee, {LHS, RHS});
19426 case WebAssembly::BI__builtin_wasm_pmin_f32x4:
19427 case WebAssembly::BI__builtin_wasm_pmin_f64x2: {
19432 return Builder.CreateCall(Callee, {LHS, RHS});
19434 case WebAssembly::BI__builtin_wasm_pmax_f32x4:
19435 case WebAssembly::BI__builtin_wasm_pmax_f64x2: {
19440 return Builder.CreateCall(Callee, {LHS, RHS});
19442 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
19443 case WebAssembly::BI__builtin_wasm_floor_f32x4:
19444 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
19445 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
19446 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
19447 case WebAssembly::BI__builtin_wasm_floor_f64x2:
19448 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
19449 case WebAssembly::BI__builtin_wasm_nearest_f64x2: {
19451 switch (BuiltinID) {
19452 case WebAssembly::BI__builtin_wasm_ceil_f32x4:
19453 case WebAssembly::BI__builtin_wasm_ceil_f64x2:
19454 IntNo = Intrinsic::ceil;
19456 case WebAssembly::BI__builtin_wasm_floor_f32x4:
19457 case WebAssembly::BI__builtin_wasm_floor_f64x2:
19458 IntNo = Intrinsic::floor;
19460 case WebAssembly::BI__builtin_wasm_trunc_f32x4:
19461 case WebAssembly::BI__builtin_wasm_trunc_f64x2:
19462 IntNo = Intrinsic::trunc;
19464 case WebAssembly::BI__builtin_wasm_nearest_f32x4:
19465 case WebAssembly::BI__builtin_wasm_nearest_f64x2:
19466 IntNo = Intrinsic::nearbyint;
19469 llvm_unreachable(
"unexpected builtin ID");
19473 return Builder.CreateCall(Callee,
Value);
19475 case WebAssembly::BI__builtin_wasm_ref_null_extern: {
19477 return Builder.CreateCall(Callee);
19479 case WebAssembly::BI__builtin_wasm_ref_null_func: {
19481 return Builder.CreateCall(Callee);
19483 case WebAssembly::BI__builtin_wasm_swizzle_i8x16: {
19487 return Builder.CreateCall(Callee, {Src, Indices});
19489 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
19490 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
19491 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
19492 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
19493 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
19494 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
19495 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
19496 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8: {
19498 switch (BuiltinID) {
19499 case WebAssembly::BI__builtin_wasm_add_sat_s_i8x16:
19500 case WebAssembly::BI__builtin_wasm_add_sat_s_i16x8:
19501 IntNo = Intrinsic::sadd_sat;
19503 case WebAssembly::BI__builtin_wasm_add_sat_u_i8x16:
19504 case WebAssembly::BI__builtin_wasm_add_sat_u_i16x8:
19505 IntNo = Intrinsic::uadd_sat;
19507 case WebAssembly::BI__builtin_wasm_sub_sat_s_i8x16:
19508 case WebAssembly::BI__builtin_wasm_sub_sat_s_i16x8:
19509 IntNo = Intrinsic::wasm_sub_sat_signed;
19511 case WebAssembly::BI__builtin_wasm_sub_sat_u_i8x16:
19512 case WebAssembly::BI__builtin_wasm_sub_sat_u_i16x8:
19513 IntNo = Intrinsic::wasm_sub_sat_unsigned;
19516 llvm_unreachable(
"unexpected builtin ID");
19521 return Builder.CreateCall(Callee, {LHS, RHS});
19523 case WebAssembly::BI__builtin_wasm_abs_i8x16:
19524 case WebAssembly::BI__builtin_wasm_abs_i16x8:
19525 case WebAssembly::BI__builtin_wasm_abs_i32x4:
19526 case WebAssembly::BI__builtin_wasm_abs_i64x2: {
19528 Value *
Neg = Builder.CreateNeg(Vec,
"neg");
19529 Constant *
Zero = llvm::Constant::getNullValue(Vec->
getType());
19530 Value *ICmp = Builder.CreateICmpSLT(Vec, Zero,
"abscond");
19531 return Builder.CreateSelect(ICmp, Neg, Vec,
"abs");
19533 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
19534 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
19535 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
19536 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
19537 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
19538 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
19539 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
19540 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
19541 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
19542 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
19543 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
19544 case WebAssembly::BI__builtin_wasm_max_u_i32x4: {
19548 switch (BuiltinID) {
19549 case WebAssembly::BI__builtin_wasm_min_s_i8x16:
19550 case WebAssembly::BI__builtin_wasm_min_s_i16x8:
19551 case WebAssembly::BI__builtin_wasm_min_s_i32x4:
19552 ICmp = Builder.CreateICmpSLT(LHS, RHS);
19554 case WebAssembly::BI__builtin_wasm_min_u_i8x16:
19555 case WebAssembly::BI__builtin_wasm_min_u_i16x8:
19556 case WebAssembly::BI__builtin_wasm_min_u_i32x4:
19557 ICmp = Builder.CreateICmpULT(LHS, RHS);
19559 case WebAssembly::BI__builtin_wasm_max_s_i8x16:
19560 case WebAssembly::BI__builtin_wasm_max_s_i16x8:
19561 case WebAssembly::BI__builtin_wasm_max_s_i32x4:
19562 ICmp = Builder.CreateICmpSGT(LHS, RHS);
19564 case WebAssembly::BI__builtin_wasm_max_u_i8x16:
19565 case WebAssembly::BI__builtin_wasm_max_u_i16x8:
19566 case WebAssembly::BI__builtin_wasm_max_u_i32x4:
19567 ICmp = Builder.CreateICmpUGT(LHS, RHS);
19570 llvm_unreachable(
"unexpected builtin ID");
19572 return Builder.CreateSelect(ICmp, LHS, RHS);
19574 case WebAssembly::BI__builtin_wasm_avgr_u_i8x16:
19575 case WebAssembly::BI__builtin_wasm_avgr_u_i16x8: {
19580 return Builder.CreateCall(Callee, {LHS, RHS});
19582 case WebAssembly::BI__builtin_wasm_q15mulr_sat_s_i16x8: {
19586 return Builder.CreateCall(Callee, {LHS, RHS});
19588 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
19589 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
19590 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
19591 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4: {
19594 switch (BuiltinID) {
19595 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_s_i16x8:
19596 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_s_i32x4:
19597 IntNo = Intrinsic::wasm_extadd_pairwise_signed;
19599 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i8x16_u_i16x8:
19600 case WebAssembly::BI__builtin_wasm_extadd_pairwise_i16x8_u_i32x4:
19601 IntNo = Intrinsic::wasm_extadd_pairwise_unsigned;
19604 llvm_unreachable(
"unexpected builtin ID");
19608 return Builder.CreateCall(Callee, Vec);
19610 case WebAssembly::BI__builtin_wasm_bitselect: {
19616 return Builder.CreateCall(Callee, {V1, V2,
C});
19618 case WebAssembly::BI__builtin_wasm_dot_s_i32x4_i16x8: {
19622 return Builder.CreateCall(Callee, {LHS, RHS});
19624 case WebAssembly::BI__builtin_wasm_popcnt_i8x16: {
19628 return Builder.CreateCall(Callee, {Vec});
19630 case WebAssembly::BI__builtin_wasm_any_true_v128:
19631 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
19632 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
19633 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
19634 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
19636 switch (BuiltinID) {
19637 case WebAssembly::BI__builtin_wasm_any_true_v128:
19638 IntNo = Intrinsic::wasm_anytrue;
19640 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
19641 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
19642 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
19643 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
19644 IntNo = Intrinsic::wasm_alltrue;
19647 llvm_unreachable(
"unexpected builtin ID");
19651 return Builder.CreateCall(Callee, {Vec});
19653 case WebAssembly::BI__builtin_wasm_bitmask_i8x16:
19654 case WebAssembly::BI__builtin_wasm_bitmask_i16x8:
19655 case WebAssembly::BI__builtin_wasm_bitmask_i32x4:
19656 case WebAssembly::BI__builtin_wasm_bitmask_i64x2: {
19660 return Builder.CreateCall(Callee, {Vec});
19662 case WebAssembly::BI__builtin_wasm_abs_f32x4:
19663 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
19666 return Builder.CreateCall(Callee, {Vec});
19668 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
19669 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
19672 return Builder.CreateCall(Callee, {Vec});
19674 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
19675 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
19676 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
19677 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
19681 switch (BuiltinID) {
19682 case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
19683 case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
19684 IntNo = Intrinsic::wasm_narrow_signed;
19686 case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
19687 case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
19688 IntNo = Intrinsic::wasm_narrow_unsigned;
19691 llvm_unreachable(
"unexpected builtin ID");
19695 return Builder.CreateCall(Callee, {Low, High});
19697 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
19698 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4: {
19701 switch (BuiltinID) {
19702 case WebAssembly::BI__builtin_wasm_trunc_sat_s_zero_f64x2_i32x4:
19703 IntNo = Intrinsic::fptosi_sat;
19705 case WebAssembly::BI__builtin_wasm_trunc_sat_u_zero_f64x2_i32x4:
19706 IntNo = Intrinsic::fptoui_sat;
19709 llvm_unreachable(
"unexpected builtin ID");
19711 llvm::Type *SrcT = Vec->
getType();
19712 llvm::Type *TruncT = SrcT->getWithNewType(Builder.getInt32Ty());
19714 Value *Trunc = Builder.CreateCall(Callee, Vec);
19715 Value *Splat = Constant::getNullValue(TruncT);
19716 return Builder.CreateShuffleVector(Trunc, Splat,
ArrayRef<int>{0, 1, 2, 3});
19718 case WebAssembly::BI__builtin_wasm_shuffle_i8x16: {
19723 while (OpIdx < 18) {
19724 std::optional<llvm::APSInt> LaneConst =
19726 assert(LaneConst &&
"Constant arg isn't actually constant?");
19727 Ops[OpIdx++] = llvm::ConstantInt::get(
getLLVMContext(), *LaneConst);
19730 return Builder.CreateCall(Callee, Ops);
19732 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
19733 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
19734 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
19735 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2: {
19740 switch (BuiltinID) {
19741 case WebAssembly::BI__builtin_wasm_relaxed_madd_f32x4:
19742 case WebAssembly::BI__builtin_wasm_relaxed_madd_f64x2:
19743 IntNo = Intrinsic::wasm_relaxed_madd;
19745 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f32x4:
19746 case WebAssembly::BI__builtin_wasm_relaxed_nmadd_f64x2:
19747 IntNo = Intrinsic::wasm_relaxed_nmadd;
19750 llvm_unreachable(
"unexpected builtin ID");
19753 return Builder.CreateCall(Callee, {A, B,
C});
19755 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i8x16:
19756 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i16x8:
19757 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i32x4:
19758 case WebAssembly::BI__builtin_wasm_relaxed_laneselect_i64x2: {
19764 return Builder.CreateCall(Callee, {A, B,
C});
19766 case WebAssembly::BI__builtin_wasm_relaxed_swizzle_i8x16: {
19770 return Builder.CreateCall(Callee, {Src, Indices});
19772 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
19773 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
19774 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
19775 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2: {
19779 switch (BuiltinID) {
19780 case WebAssembly::BI__builtin_wasm_relaxed_min_f32x4:
19781 case WebAssembly::BI__builtin_wasm_relaxed_min_f64x2:
19782 IntNo = Intrinsic::wasm_relaxed_min;
19784 case WebAssembly::BI__builtin_wasm_relaxed_max_f32x4:
19785 case WebAssembly::BI__builtin_wasm_relaxed_max_f64x2:
19786 IntNo = Intrinsic::wasm_relaxed_max;
19789 llvm_unreachable(
"unexpected builtin ID");
19792 return Builder.CreateCall(Callee, {LHS, RHS});
19794 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
19795 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
19796 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
19797 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2: {
19800 switch (BuiltinID) {
19801 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_i32x4_f32x4:
19802 IntNo = Intrinsic::wasm_relaxed_trunc_signed;
19804 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_i32x4_f32x4:
19805 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned;
19807 case WebAssembly::BI__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2:
19808 IntNo = Intrinsic::wasm_relaxed_trunc_signed_zero;
19810 case WebAssembly::BI__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2:
19811 IntNo = Intrinsic::wasm_relaxed_trunc_unsigned_zero;
19814 llvm_unreachable(
"unexpected builtin ID");
19817 return Builder.CreateCall(Callee, {Vec});
19819 case WebAssembly::BI__builtin_wasm_relaxed_q15mulr_s_i16x8: {
19823 return Builder.CreateCall(Callee, {LHS, RHS});
19825 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8: {
19830 return Builder.CreateCall(Callee, {LHS, RHS});
19832 case WebAssembly::BI__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4: {
19837 CGM.
getIntrinsic(Intrinsic::wasm_relaxed_dot_i8x16_i7x16_add_signed);
19838 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
19840 case WebAssembly::BI__builtin_wasm_relaxed_dot_bf16x8_add_f32_f32x4: {
19846 return Builder.CreateCall(Callee, {LHS, RHS, Acc});
19848 case WebAssembly::BI__builtin_wasm_table_get: {
19859 "Unexpected reference type for __builtin_wasm_table_get");
19860 return Builder.CreateCall(Callee, {Table, Index});
19862 case WebAssembly::BI__builtin_wasm_table_set: {
19874 "Unexpected reference type for __builtin_wasm_table_set");
19875 return Builder.CreateCall(Callee, {Table, Index, Val});
19877 case WebAssembly::BI__builtin_wasm_table_size: {
19881 return Builder.CreateCall(Callee,
Value);
19883 case WebAssembly::BI__builtin_wasm_table_grow: {
19896 "Unexpected reference type for __builtin_wasm_table_grow");
19898 return Builder.CreateCall(Callee, {Table, Val, NElems});
19900 case WebAssembly::BI__builtin_wasm_table_fill: {
19914 "Unexpected reference type for __builtin_wasm_table_fill");
19916 return Builder.CreateCall(Callee, {Table, Index, Val, NElems});
19918 case WebAssembly::BI__builtin_wasm_table_copy: {
19928 return Builder.CreateCall(Callee, {TableX, TableY, SrcIdx, DstIdx, NElems});
19935static std::pair<Intrinsic::ID, unsigned>
19938 unsigned BuiltinID;
19939 Intrinsic::ID IntrinsicID;
19942 static Info Infos[] = {
19943#define CUSTOM_BUILTIN_MAPPING(x,s) \
19944 { Hexagon::BI__builtin_HEXAGON_##x, Intrinsic::hexagon_##x, s },
19976#include "clang/Basic/BuiltinsHexagonMapCustomDep.def"
19977#undef CUSTOM_BUILTIN_MAPPING
19980 auto CmpInfo = [] (Info A, Info B) {
return A.BuiltinID < B.BuiltinID; };
19981 static const bool SortOnce = (llvm::sort(Infos, CmpInfo),
true);
19984 const Info *F = llvm::lower_bound(Infos, Info{BuiltinID, 0, 0}, CmpInfo);
19985 if (F == std::end(Infos) || F->BuiltinID != BuiltinID)
19986 return {Intrinsic::not_intrinsic, 0};
19988 return {F->IntrinsicID, F->VecLen};
19997 auto MakeCircOp = [
this, E](
unsigned IntID,
bool IsLoad) {
20002 llvm::Value *
Base = Builder.CreateLoad(BP);
20012 for (
unsigned i = 1, e = E->
getNumArgs(); i != e; ++i)
20018 llvm::Value *NewBase = IsLoad ? Builder.CreateExtractValue(
Result, 1)
20022 llvm::Value *RetVal =
20023 Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
20025 RetVal = Builder.CreateExtractValue(
Result, 0);
20032 auto MakeBrevLd = [
this, E](
unsigned IntID, llvm::Type *DestTy) {
20036 llvm::Value *BaseAddress =
20045 llvm::Value *DestAddress = DestAddr.
getPointer();
20050 llvm::Value *
Result = Builder.CreateCall(
20051 CGM.
getIntrinsic(IntID), {BaseAddress, EmitScalarExpr(E->getArg(2))});
20054 llvm::Value *DestVal = Builder.CreateExtractValue(
Result, 0);
20059 DestVal = Builder.CreateTrunc(DestVal, DestTy);
20061 Builder.CreateAlignedStore(DestVal, DestAddress, DestAddr.
getAlignment());
20063 return Builder.CreateExtractValue(
Result, 1);
20066 auto V2Q = [
this, VecLen] (llvm::Value *Vec) {
20067 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandvrt_128B
20068 : Intrinsic::hexagon_V6_vandvrt;
20070 {Vec, Builder.getInt32(-1)});
20072 auto Q2V = [
this, VecLen] (llvm::Value *Pred) {
20073 Intrinsic::ID ID = VecLen == 128 ? Intrinsic::hexagon_V6_vandqrt_128B
20074 : Intrinsic::hexagon_V6_vandqrt;
20076 {Pred, Builder.getInt32(-1)});
20079 switch (BuiltinID) {
20083 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
20084 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B:
20085 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
20086 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
20091 llvm::Value *PredIn = V2Q(Builder.CreateLoad(PredAddr));
20093 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1)), PredIn});
20095 llvm::Value *PredOut = Builder.CreateExtractValue(
Result, 1);
20096 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.
getPointer(),
20098 return Builder.CreateExtractValue(
Result, 0);
20103 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo:
20104 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarryo_128B:
20105 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo:
20106 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarryo_128B: {
20112 {EmitScalarExpr(E->getArg(0)), EmitScalarExpr(E->getArg(1))});
20114 llvm::Value *PredOut = Builder.CreateExtractValue(
Result, 1);
20115 Builder.CreateAlignedStore(Q2V(PredOut), PredAddr.
getPointer(),
20117 return Builder.CreateExtractValue(
Result, 0);
20120 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq:
20121 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq:
20122 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq:
20123 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq:
20124 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstoreq_128B:
20125 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorenq_128B:
20126 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentq_128B:
20127 case Hexagon::BI__builtin_HEXAGON_V6_vmaskedstorentnq_128B: {
20131 if (
auto *Cast = dyn_cast<ImplicitCastExpr>(PredOp)) {
20132 if (
Cast->getCastKind() == CK_BitCast)
20133 PredOp =
Cast->getSubExpr();
20136 for (
int i = 1, e = E->
getNumArgs(); i != e; ++i)
20141 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
20142 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
20143 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
20144 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
20145 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
20146 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
20147 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
20148 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
20149 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
20150 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
20151 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
20152 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
20153 return MakeCircOp(ID,
true);
20154 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
20155 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
20156 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
20157 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
20158 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
20159 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
20160 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
20161 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
20162 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
20163 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
20164 return MakeCircOp(ID,
false);
20165 case Hexagon::BI__builtin_brev_ldub:
20166 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr,
Int8Ty);
20167 case Hexagon::BI__builtin_brev_ldb:
20168 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr,
Int8Ty);
20169 case Hexagon::BI__builtin_brev_lduh:
20170 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr,
Int16Ty);
20171 case Hexagon::BI__builtin_brev_ldh:
20172 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr,
Int16Ty);
20173 case Hexagon::BI__builtin_brev_ldw:
20174 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr,
Int32Ty);
20175 case Hexagon::BI__builtin_brev_ldd:
20176 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr,
Int64Ty);
20189 unsigned ICEArguments = 0;
20197 if (BuiltinID == RISCVVector::BI__builtin_rvv_vget_v ||
20198 BuiltinID == RISCVVector::BI__builtin_rvv_vset_v)
20199 ICEArguments = 1 << 1;
20204 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_load)
20205 ICEArguments |= (1 << 1);
20206 if (BuiltinID == RISCV::BI__builtin_riscv_ntl_store)
20207 ICEArguments |= (1 << 2);
20209 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
20213 llvm::Value *AggValue = Builder.CreateLoad(L.getAddress(*
this));
20214 Ops.push_back(AggValue);
20219 if ((ICEArguments & (1 << i)) == 0) {
20226 Ops.push_back(llvm::ConstantInt::get(
20230 Intrinsic::ID ID = Intrinsic::not_intrinsic;
20234 constexpr unsigned RVV_VTA = 0x1;
20235 constexpr unsigned RVV_VMA = 0x2;
20236 int PolicyAttrs = 0;
20237 bool IsMasked =
false;
20241 switch (BuiltinID) {
20242 default: llvm_unreachable(
"unexpected builtin ID");
20243 case RISCV::BI__builtin_riscv_orc_b_32:
20244 case RISCV::BI__builtin_riscv_orc_b_64:
20245 case RISCV::BI__builtin_riscv_clz_32:
20246 case RISCV::BI__builtin_riscv_clz_64:
20247 case RISCV::BI__builtin_riscv_ctz_32:
20248 case RISCV::BI__builtin_riscv_ctz_64:
20249 case RISCV::BI__builtin_riscv_clmul_32:
20250 case RISCV::BI__builtin_riscv_clmul_64:
20251 case RISCV::BI__builtin_riscv_clmulh_32:
20252 case RISCV::BI__builtin_riscv_clmulh_64:
20253 case RISCV::BI__builtin_riscv_clmulr_32:
20254 case RISCV::BI__builtin_riscv_clmulr_64:
20255 case RISCV::BI__builtin_riscv_xperm4_32:
20256 case RISCV::BI__builtin_riscv_xperm4_64:
20257 case RISCV::BI__builtin_riscv_xperm8_32:
20258 case RISCV::BI__builtin_riscv_xperm8_64:
20259 case RISCV::BI__builtin_riscv_brev8_32:
20260 case RISCV::BI__builtin_riscv_brev8_64:
20261 case RISCV::BI__builtin_riscv_zip_32:
20262 case RISCV::BI__builtin_riscv_unzip_32: {
20263 switch (BuiltinID) {
20264 default: llvm_unreachable(
"unexpected builtin ID");
20266 case RISCV::BI__builtin_riscv_orc_b_32:
20267 case RISCV::BI__builtin_riscv_orc_b_64:
20268 ID = Intrinsic::riscv_orc_b;
20270 case RISCV::BI__builtin_riscv_clz_32:
20271 case RISCV::BI__builtin_riscv_clz_64: {
20273 Value *
Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
20274 if (
Result->getType() != ResultType)
20275 Result = Builder.CreateIntCast(
Result, ResultType,
true,
20279 case RISCV::BI__builtin_riscv_ctz_32:
20280 case RISCV::BI__builtin_riscv_ctz_64: {
20282 Value *
Result = Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
20283 if (
Result->getType() != ResultType)
20284 Result = Builder.CreateIntCast(
Result, ResultType,
true,
20290 case RISCV::BI__builtin_riscv_clmul_32:
20291 case RISCV::BI__builtin_riscv_clmul_64:
20292 ID = Intrinsic::riscv_clmul;
20294 case RISCV::BI__builtin_riscv_clmulh_32:
20295 case RISCV::BI__builtin_riscv_clmulh_64:
20296 ID = Intrinsic::riscv_clmulh;
20298 case RISCV::BI__builtin_riscv_clmulr_32:
20299 case RISCV::BI__builtin_riscv_clmulr_64:
20300 ID = Intrinsic::riscv_clmulr;
20304 case RISCV::BI__builtin_riscv_xperm8_32:
20305 case RISCV::BI__builtin_riscv_xperm8_64:
20306 ID = Intrinsic::riscv_xperm8;
20308 case RISCV::BI__builtin_riscv_xperm4_32:
20309 case RISCV::BI__builtin_riscv_xperm4_64:
20310 ID = Intrinsic::riscv_xperm4;
20314 case RISCV::BI__builtin_riscv_brev8_32:
20315 case RISCV::BI__builtin_riscv_brev8_64:
20316 ID = Intrinsic::riscv_brev8;
20318 case RISCV::BI__builtin_riscv_zip_32:
20319 ID = Intrinsic::riscv_zip;
20321 case RISCV::BI__builtin_riscv_unzip_32:
20322 ID = Intrinsic::riscv_unzip;
20326 IntrinsicTypes = {ResultType};
20333 case RISCV::BI__builtin_riscv_sha256sig0:
20334 ID = Intrinsic::riscv_sha256sig0;
20336 case RISCV::BI__builtin_riscv_sha256sig1:
20337 ID = Intrinsic::riscv_sha256sig1;
20339 case RISCV::BI__builtin_riscv_sha256sum0:
20340 ID = Intrinsic::riscv_sha256sum0;
20342 case RISCV::BI__builtin_riscv_sha256sum1:
20343 ID = Intrinsic::riscv_sha256sum1;
20347 case RISCV::BI__builtin_riscv_sm4ks:
20348 ID = Intrinsic::riscv_sm4ks;
20350 case RISCV::BI__builtin_riscv_sm4ed:
20351 ID = Intrinsic::riscv_sm4ed;
20355 case RISCV::BI__builtin_riscv_sm3p0:
20356 ID = Intrinsic::riscv_sm3p0;
20358 case RISCV::BI__builtin_riscv_sm3p1:
20359 ID = Intrinsic::riscv_sm3p1;
20363 case RISCV::BI__builtin_riscv_ntl_load: {
20367 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
20369 llvm::ConstantAsMetadata::get(Builder.getInt32(Mode->getZExtValue())));
20370 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
20371 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
20374 if(ResTy->isScalableTy()) {
20376 llvm::Type *ScalarTy = ResTy->getScalarType();
20377 Width = ScalarTy->getPrimitiveSizeInBits() *
20378 SVTy->getElementCount().getKnownMinValue();
20380 Width = ResTy->getPrimitiveSizeInBits();
20381 LoadInst *
Load = Builder.CreateLoad(
20384 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
20385 Load->setMetadata(
CGM.
getModule().getMDKindID(
"riscv-nontemporal-domain"),
20390 case RISCV::BI__builtin_riscv_ntl_store: {
20393 llvm::MDNode *RISCVDomainNode = llvm::MDNode::get(
20395 llvm::ConstantAsMetadata::get(Builder.getInt32(Mode->getZExtValue())));
20396 llvm::MDNode *NontemporalNode = llvm::MDNode::get(
20397 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
20399 Value *BC = Builder.CreateBitCast(
20400 Ops[0], llvm::PointerType::getUnqual(Ops[1]->
getType()),
"cast");
20402 StoreInst *
Store = Builder.CreateDefaultAlignedStore(Ops[1], BC);
20403 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, NontemporalNode);
20411#include "clang/Basic/riscv_vector_builtin_cg.inc"
20413#include "clang/Basic/riscv_sifive_vector_builtin_cg.inc"
20416 assert(ID != Intrinsic::not_intrinsic);
20419 return Builder.CreateCall(F, Ops,
"");
20426 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
20429 Intrinsic::ID ID = Intrinsic::not_intrinsic;
20431 switch (BuiltinID) {
20433 llvm_unreachable(
"unexpected builtin ID.");
20434 case LoongArch::BI__builtin_loongarch_cacop_d:
20435 ID = Intrinsic::loongarch_cacop_d;
20437 case LoongArch::BI__builtin_loongarch_cacop_w:
20438 ID = Intrinsic::loongarch_cacop_w;
20440 case LoongArch::BI__builtin_loongarch_dbar:
20441 ID = Intrinsic::loongarch_dbar;
20443 case LoongArch::BI__builtin_loongarch_break:
20444 ID = Intrinsic::loongarch_break;
20446 case LoongArch::BI__builtin_loongarch_ibar:
20447 ID = Intrinsic::loongarch_ibar;
20449 case LoongArch::BI__builtin_loongarch_movfcsr2gr:
20450 ID = Intrinsic::loongarch_movfcsr2gr;
20452 case LoongArch::BI__builtin_loongarch_movgr2fcsr:
20453 ID = Intrinsic::loongarch_movgr2fcsr;
20455 case LoongArch::BI__builtin_loongarch_syscall:
20456 ID = Intrinsic::loongarch_syscall;
20458 case LoongArch::BI__builtin_loongarch_crc_w_b_w:
20459 ID = Intrinsic::loongarch_crc_w_b_w;
20461 case LoongArch::BI__builtin_loongarch_crc_w_h_w:
20462 ID = Intrinsic::loongarch_crc_w_h_w;
20464 case LoongArch::BI__builtin_loongarch_crc_w_w_w:
20465 ID = Intrinsic::loongarch_crc_w_w_w;
20467 case LoongArch::BI__builtin_loongarch_crc_w_d_w:
20468 ID = Intrinsic::loongarch_crc_w_d_w;
20470 case LoongArch::BI__builtin_loongarch_crcc_w_b_w:
20471 ID = Intrinsic::loongarch_crcc_w_b_w;
20473 case LoongArch::BI__builtin_loongarch_crcc_w_h_w:
20474 ID = Intrinsic::loongarch_crcc_w_h_w;
20476 case LoongArch::BI__builtin_loongarch_crcc_w_w_w:
20477 ID = Intrinsic::loongarch_crcc_w_w_w;
20479 case LoongArch::BI__builtin_loongarch_crcc_w_d_w:
20480 ID = Intrinsic::loongarch_crcc_w_d_w;
20482 case LoongArch::BI__builtin_loongarch_csrrd_w:
20483 ID = Intrinsic::loongarch_csrrd_w;
20485 case LoongArch::BI__builtin_loongarch_csrwr_w:
20486 ID = Intrinsic::loongarch_csrwr_w;
20488 case LoongArch::BI__builtin_loongarch_csrxchg_w:
20489 ID = Intrinsic::loongarch_csrxchg_w;
20491 case LoongArch::BI__builtin_loongarch_csrrd_d:
20492 ID = Intrinsic::loongarch_csrrd_d;
20494 case LoongArch::BI__builtin_loongarch_csrwr_d:
20495 ID = Intrinsic::loongarch_csrwr_d;
20497 case LoongArch::BI__builtin_loongarch_csrxchg_d:
20498 ID = Intrinsic::loongarch_csrxchg_d;
20500 case LoongArch::BI__builtin_loongarch_iocsrrd_b:
20501 ID = Intrinsic::loongarch_iocsrrd_b;
20503 case LoongArch::BI__builtin_loongarch_iocsrrd_h:
20504 ID = Intrinsic::loongarch_iocsrrd_h;
20506 case LoongArch::BI__builtin_loongarch_iocsrrd_w:
20507 ID = Intrinsic::loongarch_iocsrrd_w;
20509 case LoongArch::BI__builtin_loongarch_iocsrrd_d:
20510 ID = Intrinsic::loongarch_iocsrrd_d;
20512 case LoongArch::BI__builtin_loongarch_iocsrwr_b:
20513 ID = Intrinsic::loongarch_iocsrwr_b;
20515 case LoongArch::BI__builtin_loongarch_iocsrwr_h:
20516 ID = Intrinsic::loongarch_iocsrwr_h;
20518 case LoongArch::BI__builtin_loongarch_iocsrwr_w:
20519 ID = Intrinsic::loongarch_iocsrwr_w;
20521 case LoongArch::BI__builtin_loongarch_iocsrwr_d:
20522 ID = Intrinsic::loongarch_iocsrwr_d;
20524 case LoongArch::BI__builtin_loongarch_cpucfg:
20525 ID = Intrinsic::loongarch_cpucfg;
20527 case LoongArch::BI__builtin_loongarch_asrtle_d:
20528 ID = Intrinsic::loongarch_asrtle_d;
20530 case LoongArch::BI__builtin_loongarch_asrtgt_d:
20531 ID = Intrinsic::loongarch_asrtgt_d;
20533 case LoongArch::BI__builtin_loongarch_lddir_d:
20534 ID = Intrinsic::loongarch_lddir_d;
20536 case LoongArch::BI__builtin_loongarch_ldpte_d:
20537 ID = Intrinsic::loongarch_ldpte_d;
20542 assert(ID != Intrinsic::not_intrinsic);
20545 return Builder.CreateCall(F, Ops);
Defines the clang::ASTContext interface.
#define X86_CPU_SUBTYPE(ENUM, STR)
#define X86_CPU_SUBTYPE_ALIAS(ENUM, ALIAS)
#define X86_VENDOR(ENUM, STRING)
#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)
#define X86_CPU_TYPE(ENUM, STR)
static constexpr Builtin::Info BuiltinInfo[]
static void Accumulate(SMap &SM, CFGBlock *B)
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, SpecialRegisterAccessKind AccessKind, StringRef SysReg="")
static llvm::Value * ARMMVEVectorReinterpret(CGBuilderTy &Builder, CodeGenFunction *CGF, llvm::Value *V, llvm::Type *DestType)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node.
static char bitActionToX86BTCode(BitTest::ActionKind A)
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
static Value * EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
static Value * tryUseTestFPKind(CodeGenFunction &CGF, unsigned BuiltinID, Value *V)
static llvm::Value * MVEImmediateShr(CGBuilderTy &Builder, llvm::Value *V, uint32_t Shift, bool Unsigned)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p,...
static llvm::Value * SignOrZeroExtend(CGBuilderTy &Builder, llvm::Value *V, llvm::Type *T, bool Unsigned)
static void InsertExplicitZeroOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static const ARMVectorIntrinsicInfo AArch64SMEIntrinsicMap[]
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value * > &Ops, llvm::Triple::ArchType Arch)
#define MMA_VARIANTS_B1_AND(geom, type)
static bool AArch64SISDIntrinsicsProvenSorted
static Value * EmitX86CompressExpand(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsCompress)
static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[]
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl * > &Seen)
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static std::pair< Intrinsic::ID, unsigned > getIntrinsicForHexagonNonClangBuiltin(unsigned BuiltinID)
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
#define MMA_INTR(geom_op_type, layout)
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value * > Ops)
static Value * emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
static bool AArch64SVEIntrinsicsProvenSorted
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
static const ARMVectorIntrinsicInfo * findARMVectorIntrinsicInMap(ArrayRef< ARMVectorIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, const CallExpr *E, MutableArrayRef< Value * > Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
#define MUTATE_LDBL(func)
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static Value * emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
static void InsertExplicitUndefOperand(CGBuilderTy &Builder, llvm::Type *Ty, SmallVectorImpl< Value * > &Ops)
static Value * EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
static Integer GetIntegerConstantValue(const Expr *E, ASTContext &Context)
#define MMA_VARIANTS(geom, type)
static bool AArch64SMEIntrinsicsProvenSorted
static llvm::Value * VectorZip(CGBuilderTy &Builder, llvm::Value *V0, llvm::Value *V1)
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
constexpr unsigned SVEBitsPerBlock
static std::optional< CodeGenFunction::MSVCIntrin > translateX86ToMsvcIntrin(unsigned BuiltinID)
static const std::pair< unsigned, unsigned > NEONEquivalentIntrinsicMap[]
#define NEONMAP0(NameBase)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Align Alignment)
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node,...
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value * > Ops)
static llvm::Value * ARMMVEVectorElementReverse(CGBuilderTy &Builder, llvm::Value *V, unsigned ReverseWidth)
#define MMA_SATF_VARIANTS(geom, type)
static std::optional< CodeGenFunction::MSVCIntrin > translateAarch64ToMsvcIntrin(unsigned BuiltinID)
static std::optional< CodeGenFunction::MSVCIntrin > translateArmToMsvcIntrin(unsigned BuiltinID)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap[]
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to @llvm.fabs().
#define CUSTOM_BUILTIN_MAPPING(x, s)
static Value * EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF, ArrayRef< Value * > Ops, llvm::Type *DstTy)
static bool isSpecialUnsignedMultiplySignedResult(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
static llvm::Value * emitPPCLoadReserveIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
static llvm::Value * CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * VectorUnzip(CGBuilderTy &Builder, llvm::Value *V, bool Odd)
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static llvm::ScalableVectorType * getSVEVectorForElementType(llvm::Type *EltTy)
static unsigned mutateLongDoubleBuiltin(unsigned BuiltinID)
#define INTRINSIC_WITH_CC(NAME)
static llvm::FixedVectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false, bool AllowBFloatArgsAndRet=true)
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
static llvm::Value * ARMMVEConstantSplat(CGBuilderTy &Builder, llvm::Type *VT)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value * > Ops, bool InvertLHS=false)
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size, Align AlignmentInBytes)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
SpecialRegisterAccessKind
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, unsigned BuiltinID, bool IsAddSub)
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static Value * emitBinaryExpMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID, llvm::Intrinsic::ID ConstrainedIntrinsicID)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size.
static llvm::Value * ARMMVEVectorSplat(CGBuilderTy &Builder, llvm::Value *V)
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value * > Ops)
static bool NEONSIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[]
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, SmallVectorImpl< Value * > &Ops, const CallExpr *E)
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
#define MMA_LDST(n, geom_op_type)
static Value * EmitX86vpcom(CodeGenFunction &CGF, ArrayRef< Value * > Ops, bool IsSigned)
static Value * emitFrexpBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Intrinsic::ID IntrinsicID)
@ _InterlockedExchangeAdd_rel
@ _InterlockedIncrement_acq
@ _InterlockedExchange_nf
@ _InterlockedIncrement_nf
@ _InterlockedExchange_acq
@ _InterlockedCompareExchange128_rel
@ _InterlockedCompareExchange128_acq
@ _InterlockedCompareExchange_acq
@ _InterlockedExchangeAdd_nf
@ _InterlockedCompareExchange_nf
@ _InterlockedDecrement_rel
@ _InterlockedExchangeSub
@ _InterlockedExchangeAdd_acq
@ _InterlockedIncrement_rel
@ _InterlockedCompareExchange128_nf
@ _InterlockedCompareExchange128
@ _InterlockedExchange_rel
@ _InterlockedDecrement_nf
@ _InterlockedExchangeAdd
@ _InterlockedDecrement_acq
@ _InterlockedCompareExchange_rel
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static RValue EmitCheckedUnsignedMultiplySignedResult(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID, llvm::StringRef Name="")
static Value * emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, unsigned ConstrainedIntrinsicID, llvm::Type *Ty, ArrayRef< Value * > Args)
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value * > Ops)
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
#define MMA_VARIANTS_B1_XOR(geom, type)
#define MMA_VARIANTS_I4(geom, type)
static Value * EmitX86ConvertIntToFp(CodeGenFunction &CGF, const CallExpr *E, ArrayRef< Value * > Ops, bool IsSigned)
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value * > Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static bool AArch64SIMDIntrinsicsProvenSorted
static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[]
CodeGenFunction::ComplexPairTy ComplexPairTy
llvm::Expected< T > Expected
#define ALIAS(NAME, TOK, FLAGS)
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Enumerates target-specific builtins in their own namespaces within namespace clang.
__DEVICE__ float modf(float __x, float *__iptr)
__DEVICE__ double nan(const char *)
__device__ __2f16 float __ockl_bool s
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
CharUnits getTypeAlignInChars(QualType T) const
Return the ABI-specified alignment of a (complete) type T, in characters.
unsigned getIntWidth(QualType T) const
Builtin::Context & BuiltinInfo
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize, const Expr *SizeExpr, ArrayType::ArraySizeModifier ASM, unsigned IndexTypeQuals) const
Return the unique reference to the type for a constant array of the specified element type.
QualType getObjCIdType() const
Represents the Objective-CC id type.
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error, unsigned *IntegerConstantArgs=nullptr) const
Return the type for the specified builtin.
const TargetInfo & getTargetInfo() const
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
unsigned getTargetAddressSpace(LangAS AS) const
@ GE_Missing_type
Missing a type.
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e....
llvm::StringRef getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
bool isConstWithoutErrnoAndExceptions(unsigned ID) const
Return true if this function has no side effects and doesn't read memory, except for possibly errno o...
bool isConstWithoutExceptions(unsigned ID) const
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
SourceLocation getBeginLoc() const LLVM_READONLY
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
CharUnits - This is an opaque type for sizes expressed in character units.
llvm::Align getAsAlign() const
getAsAlign - Returns Quantity as a valid llvm::Align, Beware llvm::Align assumes power of two 8-bit b...
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
static CharUnits One()
One - Construct a CharUnits quantity of one.
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
XRayInstrSet XRayInstrumentationBundle
Set of XRay instrumentation kinds to emit.
CharUnits getAlignment() const
Return the alignment of this pointer.
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Address withElementType(llvm::Type *ElemTy) const
Return address with different element type, but same pointer and alignment.
llvm::Value * getPointer() const
llvm::PointerType * getType() const
Return the type of the pointer value.
Address getAddress() const
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
llvm::StoreInst * CreateAlignedStore(llvm::Value *Val, llvm::Value *Addr, CharUnits Align, bool IsVolatile=false)
llvm::AtomicRMWInst * CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr, llvm::Value *Val, llvm::AtomicOrdering Ordering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
llvm::CallInst * CreateMemSet(Address Dest, llvm::Value *Value, llvm::Value *Size, bool IsVolatile=false)
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::LoadInst * CreateAlignedLoad(llvm::Type *Ty, llvm::Value *Addr, CharUnits Align, const llvm::Twine &Name="")
Address CreateAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::AtomicCmpXchgInst * CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New, llvm::AtomicOrdering SuccessOrdering, llvm::AtomicOrdering FailureOrdering, llvm::SyncScope::ID SSID=llvm::SyncScope::System)
Address CreateGEP(Address Addr, llvm::Value *Index, const llvm::Twine &Name="")
virtual std::string getDeviceSideName(const NamedDecl *ND)=0
Returns function or variable name on device side even if the current compilation is for host.
virtual llvm::GlobalVariable * getThrowInfo(QualType T)
All available information about a concrete callee.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
llvm::DIType * getOrCreateStandaloneType(QualType Ty, SourceLocation Loc)
Emit standalone debug info for a type.
CGFunctionInfo - Class to encapsulate the information about a function definition.
virtual void EmitGCMemmoveCollectable(CodeGen::CodeGenFunction &CGF, Address DestPtr, Address SrcPtr, llvm::Value *Size)=0
EnqueuedBlockInfo emitOpenCLEnqueuedBlock(CodeGenFunction &CGF, const Expr *E)
CallArgList - Type for representing both the value and type of arguments in a call.
void add(RValue rvalue, QualType type)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation.
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
llvm::Value * EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr)
std::pair< RValue, llvm::Value * > EmitAtomicCompareExchange(LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, llvm::AtomicOrdering Success=llvm::AtomicOrdering::SequentiallyConsistent, llvm::AtomicOrdering Failure=llvm::AtomicOrdering::SequentiallyConsistent, bool IsWeak=false, AggValueSlot Slot=AggValueSlot::ignored())
static TypeEvaluationKind getEvaluationKind(QualType T)
getEvaluationKind - Return the TypeEvaluationKind of QualType T.
llvm::Value * EmitSVEPredicateCast(llvm::Value *Pred, llvm::ScalableVectorType *VTy)
llvm::CallInst * EmitTrapCall(llvm::Intrinsic::ID IntrID)
Emit a call to trap or debugtrap and attach function attribute "trap-func-name" if specified.
llvm::Value * EmitLoongArchBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitTileslice(llvm::Value *Offset, llvm::Value *Base)
SanitizerSet SanOpts
Sanitizers enabled for this function.
RValue EmitBuiltinIsAligned(const CallExpr *E)
Emit IR for __builtin_is_aligned.
LValue EmitAggExprToLValue(const Expr *E)
EmitAggExprToLValue - Emit the computation of the specified expression of aggregate type into a tempo...
void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc, AbstractCallee AC, unsigned ParmNum)
Create a check for a function parameter that may potentially be declared as non-null.
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr)
void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
llvm::Value * EmitSVEGatherLoad(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
CleanupKind getARCCleanupKind()
Retrieves the default cleanup kind for an ARC cleanup.
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
LValue EmitLValue(const Expr *E, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitLValue - Emit code to compute a designator that specifies the location of the expression.
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm....
llvm::Value * EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
llvm::Value * EmitSEHExceptionInfo()
RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp)
Emit IR for __builtin_align_up/__builtin_align_down.
const LangOptions & getLangOpts() const
void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope, llvm::AtomicOrdering &AO, llvm::SyncScope::ID &SSID)
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
llvm::Value * EmitSMELdrStr(SVETypeFlags TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitUnreachable(SourceLocation Loc)
Emit a reached-unreachable diagnostic if Loc is valid and runtime checking is enabled.
llvm::Value * EmitSVETupleCreate(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal=false, bool IgnoreImag=false)
EmitComplexExpr - Emit the computation of the specified expression of complex type,...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::CallBase **callOrInvoke, bool IsMustTail, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type,...
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx, const llvm::ElementCount &Count)
llvm::Type * ConvertTypeForMem(QualType T)
llvm::Value * EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID, bool IsZExtReturn)
bool AlwaysEmitXRayCustomEvents() const
AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit XRay custom event handling c...
llvm::Value * EmitSVEDupX(llvm::Value *Scalar)
@ Default
! No language constraints on evaluation order.
const TargetInfo & getTarget() const
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
llvm::Value * EmitSEHExceptionCode()
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr, KnownNonNull_t IsKnownNonNull=NotKnownNonNull)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
EmitTargetBuiltinExpr - Emit the given builtin call.
void pushCleanupAfterFullExpr(CleanupKind Kind, As... A)
Queue a cleanup to be pushed after finishing the current full-expression, potentially with an active ...
RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID)
llvm::Value * EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E)
Address EmitArrayToPointerDecay(const Expr *Array, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
void EmitCheck(ArrayRef< std::pair< llvm::Value *, SanitizerMask > > Checked, SanitizerHandler Check, ArrayRef< llvm::Constant * > StaticArgs, ArrayRef< llvm::Value * > DynamicArgs)
Create a basic block that will either trap or call a handler function in the UBSan runtime with the p...
RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type, const CallExpr *TheCallExpr, bool IsDelete)
llvm::Value * EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
llvm::ScalableVectorType * getSVEType(const SVETypeFlags &TypeFlags)
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
llvm::Constant * EmitCheckSourceLocation(SourceLocation Loc)
Emit a description of a source location in a format suitable for passing to a runtime sanitizer handl...
void ErrorUnsupported(const Stmt *S, const char *Type)
ErrorUnsupported - Print out an error that codegen doesn't support the specified stmt yet.
CGDebugInfo * getDebugInfo()
Address EmitVAListRef(const Expr *E)
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
llvm::Value * EmitSVEMovl(const SVETypeFlags &TypeFlags, llvm::ArrayRef< llvm::Value * > Ops, unsigned BuiltinID)
void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty, SourceLocation Loc, SourceLocation AssumptionLoc, llvm::Value *Alignment, llvm::Value *OffsetValue=nullptr)
const TargetCodeGenInfo & getTargetHooks() const
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Type * getEltType(const SVETypeFlags &TypeFlags)
void EmitAggExpr(const Expr *E, AggValueSlot AS)
EmitAggExpr - Emit the computation of the specified expression of aggregate type.
bool ShouldXRayInstrumentFunction() const
ShouldXRayInstrument - Return true if the current function should be instrumented with XRay nop sleds...
llvm::Value * EmitSVEPMull(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned BuiltinID)
llvm::Value * EmitSMELd1St1(SVETypeFlags TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation.
RValue EmitOpenMPDevicePrintfCallExpr(const CallExpr *E)
bool IsInPreservedAIRegion
True if CodeGen currently emits code inside presereved access index region.
llvm::Value * EmitARCRetain(QualType type, llvm::Value *value)
bool AlwaysEmitXRayTypedEvents() const
AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit XRay typed event handling ...
void SetSqrtFPAccuracy(llvm::Value *Val)
Set the minimum required accuracy of the given sqrt operation based on CodeGenOpts.
llvm::Value * EmitSVEScatterStore(const SVETypeFlags &TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::CallInst * EmitNounwindRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
ASTContext & getContext() const
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
CGCallee EmitCallee(const Expr *E)
Address CreateMemTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignmen and cas...
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl)
llvm::Value * EmitSMEZero(SVETypeFlags TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * BuildVector(ArrayRef< llvm::Value * > Ops)
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::CallInst * EmitRuntimeCall(llvm::FunctionCallee callee, const Twine &name="")
llvm::Type * ConvertType(QualType T)
llvm::CallBase * EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, ArrayRef< llvm::Value * > args, const Twine &name="")
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Destroyer destroyARCStrongPrecise
void EmitARCIntrinsicUse(ArrayRef< llvm::Value * > values)
RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E)
llvm::Value * EmitSVEStructLoad(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address EmitMSVAListRef(const Expr *E)
Emit a "reference" to a __builtin_ms_va_list; this is always the value of the expression,...
llvm::Value * EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr, ArrayRef< llvm::Value * > IdxList, bool SignedIndices, bool IsSubtraction, SourceLocation Loc, const Twine &Name="")
Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to detect undefined behavior whe...
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
SmallVector< llvm::Type *, 2 > getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType, ArrayRef< llvm::Value * > Ops)
static bool hasAggregateEvaluationKind(QualType T)
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource Source=AlignmentSource::Type)
llvm::Value * EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue, llvm::Triple::ArchType Arch)
llvm::Value * EmitSVEStructStore(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Value * EmitSEHAbnormalTermination()
llvm::Value * EmitSMEReadWrite(SVETypeFlags TypeFlags, llvm::SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
AggValueSlot CreateAggTemp(QualType T, const Twine &Name="tmp", Address *Alloca=nullptr)
CreateAggTemp - Create a temporary memory object for the given aggregate type.
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
llvm::Value * EmitSVEAllTruePred(const SVETypeFlags &TypeFlags)
RValue GetUndefRValue(QualType Ty)
GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.
llvm::Type * SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags)
SVEBuiltinMemEltTy - Returns the memory element type for this memory access builtin.
llvm::LLVMContext & getLLVMContext()
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type,...
llvm::Value * EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value * > &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value * > &O, const char *name, unsigned shift=0, bool rightshift=false)
llvm::Value * EmitAnnotationCall(llvm::Function *AnnotationFn, llvm::Value *AnnotatedVal, StringRef AnnotationStr, SourceLocation Location, const AnnotateAttr *Attr)
Emit an annotation call (intrinsic).
llvm::ScalableVectorType * getSVEPredType(const SVETypeFlags &TypeFlags)
llvm::Value * EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags, SmallVectorImpl< llvm::Value * > &Ops, unsigned IntID)
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
llvm::Value * EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
This class organizes the cross-function state that is used while generating LLVM code.
llvm::Module & getModule() const
llvm::FunctionCallee CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false, bool AssumeConvergent=false)
Create or return a runtime function declaration with the specified type and name.
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
DiagnosticsEngine & getDiags() const
void ErrorUnsupported(const Stmt *S, const char *Type)
Print out an error that codegen doesn't support the specified stmt yet.
CGCUDARuntime & getCUDARuntime()
Return a reference to the configured CUDA runtime.
CodeGenTypes & getTypes()
CGOpenCLRuntime & getOpenCLRuntime()
Return a reference to the configured OpenCL runtime.
const TargetInfo & getTarget() const
const llvm::DataLayout & getDataLayout() const
void Error(SourceLocation loc, StringRef error)
Emit a general error that something can't be done.
CGCXXABI & getCXXABI() const
llvm::Constant * GetFunctionStart(const ValueDecl *Decl)
const llvm::Triple & getTriple() const
void DecorateInstructionWithTBAA(llvm::Instruction *Inst, TBAAAccessInfo TBAAInfo)
DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
llvm::Constant * CreateRuntimeVariable(llvm::Type *Ty, StringRef Name)
Create a new runtime global variable with the specified type and name.
TBAAAccessInfo getTBAAAccessInfo(QualType AccessType)
getTBAAAccessInfo - Get TBAA information that describes an access to an object of the given type.
ASTContext & getContext() const
const CodeGenOptions & getCodeGenOpts() const
StringRef getMangledName(GlobalDecl GD)
CharUnits getNaturalPointeeTypeAlignment(QualType T, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
llvm::LLVMContext & getLLVMContext()
CGObjCRuntime & getObjCRuntime()
Return a reference to the configured Objective-C runtime.
void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info, llvm::Function *F, bool IsThunk)
Set the LLVM function attributes (sext, zext, etc).
void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F)
Set the LLVM function attributes which only apply to a function definition.
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type * > Tys=std::nullopt)
ConstantAddress GetAddrOfConstantCString(const std::string &Str, const char *GlobalName=nullptr)
Returns a pointer to a character array containing the literal and a terminating '\0' character.
llvm::Type * ConvertType(QualType T)
ConvertType - Convert type T into a llvm::Type.
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
const CGFunctionInfo & arrangeBuiltinFunctionCall(QualType resultType, const CallArgList &args)
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded.
Information for lazily generating a cleanup.
FunctionArgList - Type for representing both the decl and type of parameters to a function.
llvm::Value * getBitFieldPointer() const
llvm::Value * getPointer(CodeGenFunction &CGF) const
RValue - This trivial value class is used to represent the result of an expression that is evaluated.
static RValue getIgnored()
static RValue get(llvm::Value *V)
static RValue getAggregate(Address addr, bool isVolatile=false)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
ReturnValueSlot - Contains the address where the return value of a function can be stored,...
virtual llvm::Value * encodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert the address of an instruction into a return address ...
virtual llvm::Value * decodeReturnAddress(CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const
Performs the code-generation required to convert a return address as stored by the system into the ac...
const T & getABIInfo() const
virtual int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const
Determines the DWARF register number for the stack pointer, for exception-handling purposes.
virtual llvm::Value * testFPKind(llvm::Value *V, unsigned BuiltinID, CGBuilderTy &Builder, CodeGenModule &CGM) const
Performs a target specific test of a floating point value for things like IsNaN, Infinity,...
Complex values, per C99 6.2.5p11.
Represents a concrete matrix type with constant number of rows and columns.
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
Concrete class used by the front-end to report problems and issues.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
This represents one expression.
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer,...
std::optional< llvm::APSInt > getIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return the value if this expression is a valid integer constant expression.
Expr * IgnoreParenCasts() LLVM_READONLY
Skip past any parentheses and casts which might surround this expression until reaching a fixed point...
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
Expr * IgnoreParenImpCasts() LLVM_READONLY
Skip past any parentheses and implicit casts which might surround this expression until reaching a fi...
bool EvaluateAsFloat(llvm::APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects, bool InConstantContext=false) const
EvaluateAsFloat - Return true if this is a constant which we can fold and convert to a floating point...
Expr * IgnoreParens() LLVM_READONLY
Skip past any parentheses which might surround this expression until reaching a fixed point.
@ NPC_ValueDependentIsNotNull
Specifies that a value-dependent expression should be considered to never be a null pointer constant.
ExprObjectKind getObjectKind() const
getObjectKind - The object kind that this expression produces.
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
Expr * IgnoreImpCasts() LLVM_READONLY
Skip past any implicit casts which might surround this expression until reaching a fixed point.
NullPointerConstantKind isNullPointerConstant(ASTContext &Ctx, NullPointerConstantValueDependence NPC) const
isNullPointerConstant - C99 6.3.2.3p3 - Test if this reduces down to a Null pointer constant.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
const ValueDecl * getAsBuiltinConstantDeclRef(const ASTContext &Context) const
If this expression is an unambiguous reference to a single declaration, in the style of __builtin_fun...
Represents a member of a struct/union/class.
Represents a function declaration or definition.
const ParmVarDecl * getParamDecl(unsigned i) const
Represents a prototype with parameter type info, e.g.
GlobalDecl - represents a global declaration.
const Decl * getDecl() const
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
@ Other
Other implicit parameter.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, ImplicitParamKind ParamKind)
Create implicit parameter.
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
Flags to identify the types for overloaded Neon builtins.
EltType getEltType() const
PointerType - C99 6.7.5.1 - Pointer Declarators.
QualType getPointeeType() const
A (possibly-)qualified type.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
bool isWebAssemblyFuncrefType() const
Returns true if it is a WebAssembly Funcref Type.
LangAS getAddressSpace() const
Return the address space of this type.
QualType getCanonicalType() const
bool isWebAssemblyExternrefType() const
Returns true if it is a WebAssembly Externref Type.
The collection of all-type qualifiers we support.
Flags to identify the types for overloaded SVE builtins.
bool isZExtReturn() const
bool isReverseUSDOT() const
bool isOverloadNone() const
MemEltType getMemEltType() const
bool isGatherLoad() const
bool isOverloadCvt() const
EltType getEltType() const
bool isOverloadDefault() const
bool isOverloadWhileRW() const
bool isReverseMergeAnyAccOp() const
bool isInsertOp1SVALL() const
bool isOverloadWhile() const
bool isAppendSVALL() const
bool isReverseMergeAnyBinOp() const
bool isStructStore() const
bool isTupleCreate() const
bool isGatherPrefetch() const
bool hasSplatOperand() const
MergeType getMergeType() const
bool isByteIndexed() const
bool isStructLoad() const
unsigned getSplatOperand() const
bool isScatterStore() const
bool isReverseCompare() const
Scope - A scope is a transient data structure that is used while parsing the program.
Encodes a location in the source.
SourceLocation getBeginLoc() const LLVM_READONLY
Exposes information about the current target.
TargetOptions & getTargetOpts() const
Retrieve the target options.
virtual bool hasLegalHalfType() const
Determine whether _Float16 is supported on this target.
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
bool isLittleEndian() const
unsigned getMaxOpenCLWorkGroupSize() const
virtual bool checkArithmeticFenceSupported() const
Controls if __arithmetic_fence is supported in the targeted backend.
unsigned getSuitableAlign() const
Return the alignment that is the largest alignment ever used for any scalar/SIMD data type on the tar...
virtual std::string_view getClobbers() const =0
Returns a string of target-specific clobbers, in LLVM format.
CodeObjectVersionKind CodeObjectVersion
Code object version for AMDGPU.
The base class of the type hierarchy.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
bool isBlockPointerType() const
bool isBooleanType() const
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char,...
bool isPointerType() const
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
const T * castAs() const
Member-template castAs<specific type>.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee.
bool isBitIntType() const
bool isFloatingType() const
bool isUnsignedIntegerType() const
Return true if this is an integer type that is unsigned, according to C99 6.2.5p6 [which returns true...
const T * getAs() const
Member-template getAs<specific type>'.
Represents a GCC generic vector type.
SmallVector< OSLogBufferItem, 4 > Items
unsigned char getNumArgsByte() const
unsigned char getSummaryByte() const
Defines the clang::TargetInfo interface.
@ Type
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Constant * initializationPatternFor(CodeGenModule &, llvm::Type *)
TypeEvaluationKind
The kind of evaluation to perform on values of a particular type.
@ EHCleanup
Denotes a cleanup that should run when a scope is exited using exceptional control flow (a throw stat...
constexpr XRayInstrMask Typed
constexpr XRayInstrMask Custom
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
bool Dup(InterpState &S, CodePtr OpPC)
bool Call(InterpState &S, CodePtr OpPC, const Function *Func)
bool Ret(InterpState &S, CodePtr &PC, APValue &Result)
bool Zero(InterpState &S, CodePtr OpPC)
bool Neg(InterpState &S, CodePtr OpPC)
bool Load(InterpState &S, CodePtr OpPC)
bool Cast(InterpState &S, CodePtr OpPC)
bool isa(CodeGen::Address addr)
@ OK_BitField
A bitfield object is a bitfield on a C or C++ record.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
@ C
Languages that the frontend can parse and compile.
@ Asm
Assembly: we accept this only so that we can preprocess it.
@ Result
The result type of a method or function.
U cast(CodeGen::Address addr)
YAML serialization mapping.
llvm::PointerType * VoidPtrTy
llvm::IntegerType * Int64Ty
llvm::PointerType * Int8PtrPtrTy
llvm::PointerType * ConstGlobalsPtrTy
void* in the address space for constant globals
CharUnits getIntAlign() const
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
llvm::Type * HalfTy
half, bfloat, float, double
llvm::IntegerType * SizeTy
llvm::IntegerType * Int32Ty
llvm::IntegerType * IntPtrTy
llvm::IntegerType * IntTy
int
llvm::IntegerType * Int16Ty
llvm::PointerType * Int8PtrTy
llvm::PointerType * AllocaInt8PtrTy
EvalResult is a struct with detailed info about an evaluated expression.
APValue Val
Val - This is the value the expression can be folded to.
bool has(SanitizerMask K) const
Check if a certain (single) sanitizer is enabled.
bool has(XRayInstrMask K) const
#define scalbln(__x, __y)
#define copysign(__x, __y)
#define remquo(__x, __y, __z)
#define nextafter(__x, __y)
#define nexttoward(__x, __y)
#define remainder(__x, __y)
#define fma(__x, __y, __z)